codecritique 1.0.0

package/src/utils/pr-chunking.js

@@ -0,0 +1,276 @@
+import chalk from 'chalk';
+
+/**
+ * Determines if a PR should be chunked based on estimated token usage
+ * @param {Array} prFiles - Array of PR files with diffContent and content
+ * @returns {Object} Decision object with shouldChunk flag and estimates
+ */
+export function shouldChunkPR(prFiles) {
+  // IMPORTANT: The holistic PR prompt includes BOTH full file content AND diff content
+  // for each file, plus context (code examples, guidelines, PR comments, custom docs)
+
+  // Calculate tokens for diff content
+  const diffTokens = prFiles.reduce((sum, file) => {
+    return sum + Math.ceil((file.diffContent?.length || 0) / 3);
+  }, 0);
+
+  // Calculate tokens for full file content (included in prompt for context awareness)
+  const fullContentTokens = prFiles.reduce((sum, file) => {
+    return sum + Math.ceil((file.content?.length || 0) / 3);
+  }, 0);
+
+  // Total file-related tokens (both diff AND full content are sent)
+  const fileTokens = diffTokens + fullContentTokens;
+
+  // Estimate context overhead (code examples, guidelines, PR comments, custom docs, project summary)
+  // This is typically 10-30k tokens depending on project size
+  const CONTEXT_OVERHEAD_TOKENS = 25000;
+
+  // Total estimated prompt tokens
+  const totalEstimatedTokens = fileTokens + CONTEXT_OVERHEAD_TOKENS;
+
+  // Claude's limit is 200k tokens. Leave buffer for response and safety margin.
+  // Max safe prompt size ~150k tokens to be conservative
+  const MAX_SINGLE_REVIEW_TOKENS = 100000;
+
+  const shouldChunk = totalEstimatedTokens > MAX_SINGLE_REVIEW_TOKENS || prFiles.length > 30;
+
+  console.log(
+    chalk.gray(
+      `  Token breakdown: ${diffTokens} diff + ${fullContentTokens} full content + ${CONTEXT_OVERHEAD_TOKENS} context overhead = ${totalEstimatedTokens} total`
+    )
+  );
+
+  return {
+    shouldChunk,
+    estimatedTokens: totalEstimatedTokens,
+    diffTokens,
+    fullContentTokens,
+    contextOverhead: CONTEXT_OVERHEAD_TOKENS,
+    recommendedChunks: Math.ceil(totalEstimatedTokens / 35000), // More aggressive chunking
+  };
+}
+
+/**
+ * Chunks PR files into manageable groups based on token limits and logical grouping
+ * @param {Array} prFiles - Array of PR files with diffContent and content
+ * @param {number} maxTokensPerChunk - Maximum tokens per chunk
+ * @returns {Array} Array of chunks with files and metadata
+ */
+export function chunkPRFiles(prFiles, maxTokensPerChunk = 35000) {
+  // Calculate change complexity for each file (works for any language)
+  // IMPORTANT: Token estimate must include BOTH diff AND full content since both are sent
+  const filesWithMetrics = prFiles.map((file) => ({
+    ...file,
+    changeSize: calculateChangeSize(file.diffContent),
+    fileComplexity: calculateFileComplexity(file),
+    // Estimate tokens for BOTH diff content AND full file content (both are included in prompt)
+    estimatedTokens: Math.ceil((file.diffContent?.length || 0) / 3) + Math.ceil((file.content?.length || 0) / 3),
+  }));
+
+  // Sort by directory + change importance for logical grouping
+  const sortedFiles = filesWithMetrics.sort((a, b) => {
+    const dirA = getDirectoryDepth(a.filePath);
+    const dirB = getDirectoryDepth(b.filePath);
+
+    // Primary: Directory structure (keep related files together)
+    if (dirA !== dirB) return dirA.localeCompare(dirB);
+
+    // Secondary: Change importance (larger changes first)
+    return b.changeSize - a.changeSize;
+  });
+
+  // Chunk files based on token budget
+  const chunks = [];
+  let currentChunk = [];
+  let currentTokens = 0;
+
+  for (const file of sortedFiles) {
+    // Start new chunk if adding this file exceeds budget
+    if (currentTokens + file.estimatedTokens > maxTokensPerChunk && currentChunk.length > 0) {
+      chunks.push({
+        files: [...currentChunk],
+        totalTokens: currentTokens,
+        chunkId: chunks.length + 1,
+      });
+      currentChunk = [];
+      currentTokens = 0;
+    }
+
+    currentChunk.push(file);
+    currentTokens += file.estimatedTokens;
+  }
+
+  // Add final chunk
+  if (currentChunk.length > 0) {
+    chunks.push({
+      files: [...currentChunk],
+      totalTokens: currentTokens,
+      chunkId: chunks.length + 1,
+    });
+  }
+
+  return chunks;
+}
+
+/**
+ * Language-agnostic change size calculation
+ * @param {string} diffContent - The diff content
+ * @returns {number} Total number of additions and deletions
+ */
+function calculateChangeSize(diffContent) {
+  if (!diffContent) return 0;
+  const lines = diffContent.split('\n');
+  const additions = lines.filter((line) => line.startsWith('+')).length;
+  const deletions = lines.filter((line) => line.startsWith('-')).length;
+  return additions + deletions;
+}
+
+/**
+ * Language-agnostic file complexity scoring
+ * @param {Object} file - File object with filePath and diffContent
+ * @returns {number} Complexity score
+ */
+function calculateFileComplexity(file) {
+  let complexity = 0;
+
+  // File size factor
+  complexity += Math.min(file.diffContent ? file.diffContent.length / 1000 : 0, 20);
+
+  // Path-based heuristics (works for any language)
+  const path = file.filePath.toLowerCase();
+  if (path.includes('/src/') || path.includes('/lib/')) complexity += 10;
+  if (path.includes('/test/') || path.includes('/spec/')) complexity += 5;
+  if (path.includes('/config/') || path.includes('/settings/')) complexity += 8;
+  if (path.includes('/main.') || path.includes('/index.')) complexity += 15;
+
+  // Change type heuristics
+  if (file.diffContent) {
+    if (file.diffContent.includes('new file mode')) complexity += 12;
+    if (file.diffContent.includes('deleted file mode')) complexity += 8;
+  }
+
+  return complexity;
+}
+
+/**
+ * Gets directory path for grouping related files
+ * @param {string} filePath - The file path
+ * @returns {string} Directory path without filename
+ */
+function getDirectoryDepth(filePath) {
+  return filePath.split('/').slice(0, -1).join('/'); // Directory path without filename
+}
+
+/**
+ * Combines results from multiple chunk reviews into a single result
+ * @param {Array} chunkResults - Array of chunk review results
+ * @param {number} totalFiles - Total number of files in the PR
+ * @returns {Object} Combined result object
+ */
+export function combineChunkResults(chunkResults, totalFiles) {
+  const combinedResult = {
+    success: true,
+    results: [],
+    prContext: {
+      totalFiles: totalFiles,
+      chunkedReview: true,
+      chunks: chunkResults.length,
+    },
+  };
+
+  // Combine file-specific results
+  chunkResults.forEach((chunkResult, chunkIndex) => {
+    if (chunkResult.success && chunkResult.results) {
+      chunkResult.results.forEach((fileResult) => {
+        // Add chunk context to each result
+        const enhancedResult = {
+          ...fileResult,
+          chunkInfo: {
+            chunkNumber: chunkIndex + 1,
+            totalChunks: chunkResults.length,
+          },
+        };
+        combinedResult.results.push(enhancedResult);
+      });
+    }
+  });
+
+  // Create combined summary
+  combinedResult.combinedSummary = createCombinedSummary(chunkResults);
+
+  // Detect and merge cross-chunk issues
+  combinedResult.crossChunkIssues = detectCrossChunkIssues(chunkResults);
+
+  console.log(chalk.green(`✅ Combined results from ${chunkResults.length} chunks: ${combinedResult.results.length} file reviews`));
+
+  return combinedResult;
+}
+
+/**
+ * Creates a summary from combined chunk results
+ * @param {Array} chunkResults - Array of chunk review results
+ * @returns {string} Combined summary text
+ */
+function createCombinedSummary(chunkResults) {
+  const totalIssues = chunkResults.reduce((sum, chunk) => {
+    if (!chunk.results) return sum;
+    return (
+      sum +
+      chunk.results.reduce((fileSum, file) => {
+        return fileSum + (file.results?.issues?.length || 0);
+      }, 0)
+    );
+  }, 0);
+
+  const successfulChunks = chunkResults.filter((c) => c.success).length;
+
+  return `Chunked PR review completed: ${successfulChunks}/${chunkResults.length} chunks processed successfully. Total issues found: ${totalIssues}. Review performed in parallel chunks to optimize token usage.`;
+}
+
+/**
+ * Detects issues that span across multiple chunks
+ * @param {Array} chunkResults - Array of chunk review results
+ * @returns {Array} Array of cross-chunk issues
+ */
+function detectCrossChunkIssues(chunkResults) {
+  const crossChunkIssues = [];
+
+  // Simple heuristic: Look for similar issues across chunks that might indicate patterns
+  const allIssues = chunkResults.flatMap(
+    (chunk) =>
+      chunk.results?.flatMap((file) =>
+        (file.results?.issues || []).map((issue) => ({
+          ...issue,
+          chunkId: chunk.chunkId,
+          filePath: file.filePath,
+        }))
+      ) || []
+  );
+
+  // Group by issue type and description similarity
+  const issueGroups = new Map();
+  allIssues.forEach((issue) => {
+    const key = `${issue.type}-${issue.description ? issue.description.substring(0, 50) : ''}`;
+    if (!issueGroups.has(key)) {
+      issueGroups.set(key, []);
+    }
+    issueGroups.get(key).push(issue);
+  });
+
+  // Identify patterns that appear across multiple chunks
+  issueGroups.forEach((issues) => {
+    const uniqueChunks = new Set(issues.map((i) => i.chunkId));
+    if (uniqueChunks.size > 1) {
+      crossChunkIssues.push({
+        type: 'pattern',
+        severity: 'medium',
+        description: `Similar issue pattern detected across ${uniqueChunks.size} chunks: ${issues[0].description || 'Pattern issue'}`,
+        affectedFiles: issues.map((i) => i.filePath),
+        suggestion: `This issue appears in multiple parts of the PR. Consider addressing it consistently across all affected files.`,
+      });
+    }
+  });
+
+  return crossChunkIssues;
+}

package/src/utils/string-utils.js

@@ -0,0 +1,28 @@
+/**
+ * String Utilities Module
+ *
+ * This module provides utilities for string manipulation, formatting,
+ * and text processing operations.
+ */
+
+/**
+ * Slugify text for use in IDs and URLs
+ *
+ * @param {string} text - The text to slugify
+ * @returns {string} A slugified string safe for use in IDs and URLs
+ *
+ * @example
+ * slugify('Hello World!'); // 'hello-world'
+ * slugify('My Component Name'); // 'my-component-name'
+ * slugify('  Multiple   Spaces  '); // 'multiple-spaces'
+ */
+export function slugify(text) {
+  if (!text) return '';
+  return text
+    .toString()
+    .toLowerCase()
+    .trim()
+    .replace(/\s+/g, '-') // Replace spaces with -
+    .replace(/[^\w-]+/g, '') // Remove all non-word chars
+    .replace(/--+/g, '-'); // Replace multiple - with single -
+}

package/src/zero-shot-classifier-open.js

@@ -0,0 +1,392 @@
+/**
+ * Open-ended Zero-Shot Classification Module
+ *
+ * This module provides zero-shot classification without predefined categories,
+ * allowing it to detect any technology or framework mentioned in the text.
+ */
+
+import { env, pipeline } from '@huggingface/transformers';
+import * as linguistLanguages from 'linguist-languages';
+import { LRUCache } from 'lru-cache';
+import stopwords from 'stopwords-iso/stopwords-iso.json' with { type: 'json' };
+import techKeywords from './technology-keywords.json' with { type: 'json' };
+import { truncateToTokenLimit } from './utils/mobilebert-tokenizer.js';
+
+// Configure Transformers.js environment
+env.allowLocalModels = false;
+env.useBrowserCache = false;
+
+/**
+ * OpenZeroShotClassifier for unrestricted technology detection
+ */
+class OpenZeroShotClassifier {
+  constructor() {
+    this.classifier = null;
+    this.initializationPromise = null;
+    this.cache = new LRUCache({
+      max: 100,
+      ttl: 1000 * 60 * 60, // 1 hour TTL
+    });
+    this.isInitialized = false;
+
+    // Common words to exclude from technology detection
+    // Use English stopwords from stopwords-iso
+    this.commonWords = new Set(stopwords.en || []);
+
+    // Add additional technical context words that are too generic
+    const additionalCommonWords = [
+      'system',
+      'modern',
+      'architecture',
+      'stack',
+      'features',
+      'data',
+      'service',
+      'tools',
+      'runtime',
+      'apps',
+      'workloads',
+      'pipeline',
+      'builds',
+      'team',
+      'interfaces',
+      'queries',
+      'computing',
+      'database',
+      'processing',
+      'stream',
+      'analytics',
+      'infrastructure',
+      'runs',
+      'orchestration',
+      'mesh',
+      'experimenting',
+      'desktop',
+      'entire',
+      'reproducible',
+      'migrating',
+      'temporal',
+      'distributed',
+      'graph',
+      'high-performance',
+      'real-time',
+      'reactive',
+      'frontend',
+      'instead',
+      'legacy',
+      'fast',
+      'slow',
+      'quick',
+      'easy',
+      'hard',
+      'simple',
+      'complex',
+      'basic',
+      'advanced',
+      'beginner',
+      'intermediate',
+      'expert',
+      'professional',
+    ];
+
+    // Add the additional words to the stopwords set
+    additionalCommonWords.forEach((word) => this.commonWords.add(word));
+
+    // Build technology patterns from loaded keywords
+    this.techPatterns = this.buildTechPatterns();
+
+    // Build a set of all known technologies for quick lookup
+    this.knownTechnologies = this.buildKnownTechnologies();
+  }
+
+  /**
+   * Initialize the zero-shot classification pipeline (singleton pattern)
+   */
+  async initialize() {
+    // If already initialized, return immediately
+    if (this.isInitialized) return;
+
+    // If currently initializing, wait for the existing initialization
+    if (this.initializationPromise) {
+      return await this.initializationPromise;
+    }
+
+    // Start initialization
+    this.initializationPromise = this._doInitialize();
+
+    try {
+      await this.initializationPromise;
+    } finally {
+      // Clean up the promise after initialization (success or failure)
+      this.initializationPromise = null;
+    }
+  }
+
+  async _doInitialize() {
+    try {
+      console.log('Initializing open-ended zero-shot classifier...');
+
+      this.classifier = await pipeline('zero-shot-classification', 'Xenova/mobilebert-uncased-mnli', {
+        quantized: true,
+      });
+
+      this.isInitialized = true;
+      console.log('✓ Open-ended zero-shot classifier initialized successfully');
+    } catch (error) {
+      console.error('Error initializing classifier:', error);
+      this.isInitialized = false;
+      throw error;
+    }
+  }
+
+  /**
+   * Build technology patterns from keywords JSON
+   */
+  buildTechPatterns() {
+    const patterns = [
+      /\b(\w+\.js)\b/gi, // Matches *.js frameworks
+      /\b(\w+\.py)\b/gi, // Matches *.py libraries
+      /\b([A-Z](?:[a-z]*[A-Z])*[a-z]*)\b/g, // CamelCase (React, FastAPI)
+      /\b([a-z]+(?:-[a-z]+)+)\b/gi, // kebab-case (scikit-learn, styled-components)
+    ];
+
+    // Add dynamic patterns from linguist languages
+    for (const [, langData] of Object.entries(linguistLanguages)) {
+      if (langData.aliases) {
+        langData.aliases.forEach((alias) => {
+          patterns.push(new RegExp(`\\b${this.escapeRegex(alias)}\\b`, 'gi'));
+        });
+      }
+    }
+
+    return patterns;
+  }
+
+  /**
+   * Build a set of all known technologies
+   */
+  buildKnownTechnologies() {
+    const techs = new Set();
+
+    // Add all technologies from JSON file
+    const addTechsFromObject = (obj) => {
+      for (const value of Object.values(obj)) {
+        if (Array.isArray(value)) {
+          value.forEach((tech) => techs.add(tech.toLowerCase()));
+        } else if (typeof value === 'object') {
+          addTechsFromObject(value);
+        }
+      }
+    };
+
+    addTechsFromObject(techKeywords);
+
+    // Add languages from linguist
+    for (const [langName, langData] of Object.entries(linguistLanguages)) {
+      techs.add(langName.toLowerCase());
+      if (langData.aliases) {
+        langData.aliases.forEach((alias) => techs.add(alias.toLowerCase()));
+      }
+    }
+
+    return techs;
+  }
+
+  /**
+   * Escape regex special characters
+   */
+  escapeRegex(str) {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  }
+
+  /**
+   * Extract potential technology candidates from text
+   */
+  extractTechnologyCandidates(text) {
+    const candidates = new Set();
+
+    // Look for known technologies
+    for (const tech of this.knownTechnologies) {
+      // Create regex for exact word boundary matching
+      const regex = new RegExp(`\\b${this.escapeRegex(tech)}\\b`, 'i');
+      if (regex.test(text)) {
+        candidates.add(tech);
+      }
+    }
+
+    // Extract using patterns
+    for (const pattern of this.techPatterns) {
+      const matches = text.matchAll(pattern);
+      for (const match of matches) {
+        const candidate = match[1] || match[0];
+        if (candidate.length > 2 && candidate.length < 30 && !this.commonWords.has(candidate.toLowerCase())) {
+          candidates.add(candidate);
+        }
+      }
+    }
+
+    // Extract capitalized words that might be technologies
+    const sentences = text.split(/[.!?]+/);
+    for (const sentence of sentences) {
+      const words = sentence.trim().split(/\s+/);
+      for (let i = 0; i < words.length; i++) {
+        const word = words[i].replace(/[.,;:!?'"()[\]{}]/g, '');
+
+        // Skip if it's a common word
+        if (this.commonWords.has(word.toLowerCase())) continue;
+
+        // Check if word is capitalized and not at sentence start
+        if (i > 0 && /^[A-Z][a-zA-Z]+/.test(word) && word.length > 2 && word.length < 20) {
+          candidates.add(word);
+        }
+
+        // Also check for acronyms
+        if (/^[A-Z]{2,6}$/.test(word)) {
+          candidates.add(word);
+        }
+      }
+    }
+
+    return Array.from(candidates);
+  }
+
+  /**
+   * Classify if the text is about each candidate technology
+   */
+  async classifyTechnologies(text, minConfidence = 0.3) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+
+    const cacheKey = `tech:${text.substring(0, 100)}`;
+    const cached = this.cache.get(cacheKey);
+    if (cached) {
+      return cached;
+    }
+
+    try {
+      // Extract technology candidates
+      const candidates = this.extractTechnologyCandidates(text);
+
+      if (candidates.length === 0) {
+        return [];
+      }
+
+      // Truncate text using exact token counting to avoid MobileBERT's 512 token limit
+      const truncatedText = await truncateToTokenLimit(text, 450); // Conservative limit
+
+      // Create hypotheses for each candidate
+      const hypotheses = candidates.map((tech) => `This text is about ${tech}`);
+
+      // Classify
+      const result = await this.classifier(truncatedText, hypotheses, {
+        multi_label: true,
+      });
+
+      // Process results
+      const classifications = [];
+      for (let i = 0; i < result.labels.length; i++) {
+        if (result.scores[i] >= minConfidence) {
+          // Extract technology name from hypothesis
+          const tech = result.labels[i].replace('This text is about ', '');
+          classifications.push({
+            technology: tech,
+            confidence: result.scores[i],
+          });
+        }
+      }
+
+      // Sort by confidence
+      classifications.sort((a, b) => b.confidence - a.confidence);
+
+      this.cache.set(cacheKey, classifications);
+      return classifications;
+    } catch (error) {
+      console.error('Error in technology classification:', error);
+      return [];
+    }
+  }
+
+  /**
+   * Classify the general area/domain of the documentation
+   */
+  async classifyDomain(text, minConfidence = 0.3) {
+    if (!this.isInitialized) {
+      await this.initialize();
+    }
+
+    const cacheKey = `domain:${text.substring(0, 100)}`;
+    const cached = this.cache.get(cacheKey);
+    if (cached) {
+      return cached;
+    }
+
+    try {
+      // Use exact token counting to avoid MobileBERT's 512 token limit
+      const truncatedText = await truncateToTokenLimit(text, 450);
+
+      // Open-ended domain hypotheses
+      const domainHypotheses = [
+        'This is frontend/UI documentation',
+        'This is backend/server documentation',
+        'This is database documentation',
+        'This is DevOps/infrastructure documentation',
+        'This is mobile app documentation',
+        'This is data science/ML documentation',
+        'This is API documentation',
+        'This is security documentation',
+        'This is testing documentation',
+        'This is architecture documentation',
+        'This is getting started/setup documentation',
+        'This is configuration documentation',
+        'This is deployment documentation',
+        'This is troubleshooting documentation',
+        'This is reference documentation',
+        'This is tutorial documentation',
+        'This is best practices documentation',
+        'This is changelog/release notes',
+      ];
+
+      const result = await this.classifier(truncatedText, domainHypotheses, {
+        multi_label: true,
+      });
+
+      // Process results
+      const classifications = [];
+      for (let i = 0; i < result.labels.length; i++) {
+        if (result.scores[i] >= minConfidence) {
+          classifications.push({
+            domain: result.labels[i].replace('This is ', '').replace(' documentation', ''),
+            confidence: result.scores[i],
+          });
+        }
+      }
+
+      // Sort by confidence
+      classifications.sort((a, b) => b.confidence - a.confidence);
+
+      this.cache.set(cacheKey, classifications);
+      return classifications;
+    } catch (error) {
+      console.error('Error in domain classification:', error);
+      return [];
+    }
+  }
+
+  /**
+   * Get a summary classification of the text
+   */
+  async classifyDocument(text) {
+    const [technologies, domains] = await Promise.all([this.classifyTechnologies(text), this.classifyDomain(text)]);
+
+    return {
+      technologies,
+      domains,
+      primaryTechnology: technologies[0]?.technology || 'Unknown',
+      primaryDomain: domains[0]?.domain || 'general',
+    };
+  }
+}
+
+// Export singleton instance
+export const openClassifier = new OpenZeroShotClassifier();