codecritique 1.0.0

package/src/utils/language-detection.js

@@ -0,0 +1,170 @@
+/**
+ * Language Detection Module
+ *
+ * This module provides utilities for detecting programming languages
+ * and file types from file extensions and content analysis.
+ */
+
+import path from 'path';
+import { EXTENSION_TO_LANGUAGE_MAP, ALL_SUPPORTED_EXTENSIONS } from './constants.js';
+
+/**
+ * Detect programming language from file extension
+ *
+ * @param {string} extension - File extension (including the dot)
+ * @returns {string|null} Detected language or null if unknown
+ *
+ * @example
+ * const language = detectLanguageFromExtension('.ts');
+ * // Returns: 'typescript'
+ */
+export function detectLanguageFromExtension(extension) {
+  // Normalize extension to lowercase with leading dot
+  const normalizedExt = extension.toLowerCase();
+  if (!normalizedExt.startsWith('.')) {
+    extension = `.${normalizedExt}`;
+  } else {
+    extension = normalizedExt;
+  }
+
+  // Check if the extension is supported
+  if (!ALL_SUPPORTED_EXTENSIONS.includes(extension)) {
+    return 'unknown';
+  }
+
+  // Use the centralized extension-to-language mapping from constants
+  return EXTENSION_TO_LANGUAGE_MAP[extension] || 'unknown';
+}
+
+/**
+ * Detect file type and framework from file path and content
+ *
+ * @param {string} filePath - Path to the file
+ * @param {string} content - Content of the file (optional)
+ * @returns {Object} File type information including language, framework, and flags
+ *
+ * @example
+ * const fileInfo = detectFileType('src/components/Button.tsx', 'import React from "react"');
+ * // Returns: { path: '...', extension: '.tsx', language: 'typescript', framework: 'react', ... }
+ */
+export function detectFileType(filePath, content = '') {
+  // Get file extension and base name
+  const extension = path.extname(filePath);
+  const baseName = path.basename(filePath);
+
+  // Detect language from extension
+  const language = detectLanguageFromExtension(extension);
+
+  // Initialize result object
+  const result = {
+    path: filePath,
+    extension,
+    language,
+    type: 'unknown',
+    framework: null,
+    isConfig: false,
+    isTest: false,
+    isTypeDefinition: false,
+  };
+
+  // Detect file type based on name patterns
+  if (baseName.endsWith('.d.ts')) {
+    result.type = 'type-definition';
+    result.isTypeDefinition = true;
+  } else if (baseName.match(/\.test\.|\.spec\.|_test\.|_spec\./)) {
+    result.type = 'test';
+    result.isTest = true;
+  } else if (baseName.match(/^test.*\.|^spec.*\./)) {
+    result.type = 'test';
+    result.isTest = true;
+  } else if (baseName.match(/config|conf|settings|\.rc$/)) {
+    result.type = 'config';
+    result.isConfig = true;
+  } else if (language) {
+    result.type = language;
+  }
+
+  // If content is provided, perform deeper analysis
+  if (content && content.length > 0) {
+    // Detect React
+    if (
+      extension === '.jsx' ||
+      extension === '.tsx' ||
+      content.includes('import React') ||
+      content.includes('from "react"') ||
+      content.includes("from 'react'")
+    ) {
+      result.framework = 'react';
+
+      // Check for specific React patterns
+      if (content.includes('useState') || content.includes('useEffect') || content.includes('useContext')) {
+        result.isHook = content.match(/^\s*function\s+use[A-Z]/m) !== null;
+        result.isComponent = content.match(/^\s*function\s+[A-Z]/m) !== null || content.match(/^\s*const\s+[A-Z]\w+\s*=\s*\(/m) !== null;
+      }
+    }
+
+    // Detect Vue
+    else if (extension === '.vue' || (content.includes('<template>') && content.includes('<script>'))) {
+      result.framework = 'vue';
+    }
+
+    // Detect Angular
+    else if (
+      content.includes('@Component') ||
+      content.includes('@NgModule') ||
+      content.includes('from "@angular/core"') ||
+      content.includes("from '@angular/core'")
+    ) {
+      result.framework = 'angular';
+    }
+
+    // Detect Express.js
+    else if (
+      content.includes('express()') ||
+      content.includes('require("express")') ||
+      content.includes("require('express')") ||
+      content.includes('from "express"') ||
+      content.includes("from 'express'")
+    ) {
+      result.framework = 'express';
+    }
+
+    // Detect Next.js
+    else if (
+      content.includes('from "next"') ||
+      content.includes("from 'next'") ||
+      content.includes('next/app') ||
+      content.includes('next/document')
+    ) {
+      result.framework = 'nextjs';
+    }
+
+    // Detect Django (Python)
+    else if (language === 'python' && (content.includes('from django') || content.includes('import django'))) {
+      result.framework = 'django';
+    }
+
+    // Detect Flask (Python)
+    else if (language === 'python' && (content.includes('from flask import') || content.includes('import flask'))) {
+      result.framework = 'flask';
+    }
+
+    // Detect Rails (Ruby)
+    else if (language === 'ruby' && (content.includes('Rails') || content.includes('ActiveRecord'))) {
+      result.framework = 'rails';
+    }
+
+    // Detect Spring (Java)
+    else if (
+      language === 'java' &&
+      (content.includes('@Controller') ||
+        content.includes('@Service') ||
+        content.includes('@Repository') ||
+        content.includes('@SpringBootApplication'))
+    ) {
+      result.framework = 'spring';
+    }
+  }
+
+  return result;
+}

package/src/utils/logging.js

@@ -0,0 +1,24 @@
+/**
+ * Logging Module
+ *
+ * This module provides debugging and logging utilities with support
+ * for environment-based and command-line argument-based log level control.
+ */
+
+import chalk from 'chalk';
+
+/**
+ * Debug function for conditional logging based on environment variables and command line arguments
+ *
+ * @param {string} message - Debug message to log
+ *
+ * @example
+ * debug('Processing file: example.js');
+ * // Only logs if DEBUG=true, VERBOSE=true, or --verbose flag is present
+ */
+export function debug(message) {
+  const DEBUG = process.env.DEBUG || false;
+  if (DEBUG || process.env.VERBOSE === 'true' || process.argv.includes('--verbose')) {
+    console.log(chalk.cyan(`[DEBUG] ${message}`));
+  }
+}

package/src/utils/markdown.js

@@ -0,0 +1,132 @@
+/**
+ * Markdown Processing Module
+ *
+ * This module provides utilities for processing markdown content,
+ * including chunk extraction, heading analysis, and content parsing.
+ */
+
+import path from 'path';
+
+/**
+ * Extracts chunks from Markdown content based on H2 and H3 headings,
+ * and also extracts the first H1 heading as the document title.
+ *
+ * @param {string} filePath - The absolute path to the file
+ * @param {string} content - The Markdown content of the file
+ * @param {string} relativePath - The relative path of the file
+ * @returns {Object} An object containing `chunks` (Array) and `documentH1` (string|null).
+ *                   Each chunk object contains:
+ *                          `content`, `heading` (H2/H3 text),
+ *                          `original_document_path`, `start_line_in_doc`, `language`.
+ *
+ * @example
+ * const result = extractMarkdownChunks('/path/to/file.md', '# Title\n## Section\nContent...', 'docs/file.md');
+ * // Returns: { chunks: [{ content: '...', heading: 'Section', ... }], documentH1: 'Title' }
+ */
+export function extractMarkdownChunks(filePath, content, relativePath) {
+  const chunks = [];
+  let documentH1 = null;
+  if (!content || typeof content !== 'string') return { chunks, documentH1 };
+
+  const lines = content.split('\n');
+  let currentChunkLines = [];
+  let currentH2H3Heading = null; // Stores the H2 or H3 heading for the current chunk
+  let chunkStartLine = 1;
+  let inCodeBlock = false;
+  let h1Found = false;
+  let linesProcessedForH1 = 0; // Debug counter
+
+  const h1Regex = /^#\s*(.*)/; // Regex for H1 (allow zero or more spaces after #)
+  const h2h3Regex = /^(##|###)\s+(.*)/; // Regex for H2 or H3
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const trimmedLine = line.trim();
+
+    if (trimmedLine.startsWith('```')) {
+      inCodeBlock = !inCodeBlock;
+    }
+
+    if (!h1Found && linesProcessedForH1 < 5) {
+      linesProcessedForH1++;
+      // Check for H1 heading in first few lines
+      if (filePath.includes('README.md') || filePath.includes('RUNBOOK.md')) {
+        // Log only for specific files to reduce noise
+        console.log(`[extractMarkdownChunks DEBUG] File: ${filePath}, Line ${i + 1} (trimmed): "${trimmedLine}", Attempting H1 match.`);
+      }
+      const h1Match = trimmedLine.match(h1Regex);
+      if (h1Match) {
+        documentH1 = h1Match[1].trim();
+        h1Found = true;
+        console.log(`[extractMarkdownChunks DEBUG] H1 FOUND for ${filePath}: "${documentH1}" on line ${i + 1}`);
+      } else if (filePath.includes('README.md') || filePath.includes('RUNBOOK.md')) {
+        if (linesProcessedForH1 <= 5 && trimmedLine.startsWith('#')) {
+          // If it starts with # but didn't match
+          console.log(
+            `[extractMarkdownChunks DEBUG] File: ${filePath}, Line ${i + 1}: Starts with # but H1Regex DID NOT match "${trimmedLine}"`
+          );
+        }
+      }
+    }
+
+    const h2h3Match = !inCodeBlock && trimmedLine.match(h2h3Regex);
+
+    if (h2h3Match) {
+      // Found an H2 or H3 heading, finalize the previous chunk if it has content
+      if (currentChunkLines.length > 0 && currentChunkLines.join('\n').trim().length > 0) {
+        chunks.push({
+          content: currentChunkLines.join('\n').trim(),
+          heading: currentH2H3Heading, // Heading of the *previous* H2/H3 chunk
+          original_document_path: relativePath,
+          start_line_in_doc: chunkStartLine,
+          language: 'markdown',
+        });
+      }
+      // Start a new H2/H3 chunk
+      currentH2H3Heading = h2h3Match[2].trim();
+      currentChunkLines = [line]; // Include H2/H3 heading line in the new chunk's content
+      chunkStartLine = i + 1;
+    } else {
+      // Not an H1 or H2/H3 heading line (or H1 already found), add to current chunk
+      // This also correctly captures content before the first H2/H3 heading (under an H1 or if no H1).
+      currentChunkLines.push(line);
+    }
+  }
+
+  // Add the last processed chunk if it has content
+  if (currentChunkLines.length > 0 && currentChunkLines.join('\n').trim().length > 0) {
+    chunks.push({
+      content: currentChunkLines.join('\n').trim(),
+      heading: currentH2H3Heading, // H2/H3 heading of the last chunk
+      original_document_path: relativePath,
+      start_line_in_doc: chunkStartLine,
+      language: 'markdown',
+    });
+  }
+
+  // If no H2/H3 chunks were created (e.g., file has only H1 and paragraphs, or just paragraphs)
+  // treat the whole file content (that wasn't part of H1 line itself if H1 was first line) as a single chunk.
+  if (chunks.length === 0 && content.trim().length > 0) {
+    let initialContent = content.trim();
+    // If H1 was the very first line and we captured it, remove it from this single chunk content
+    if (documentH1 && lines.length > 0 && lines[0].trim().match(h1Regex)) {
+      initialContent = lines.slice(1).join('\n').trim();
+    }
+    if (initialContent.length > 0) {
+      chunks.push({
+        content: initialContent,
+        heading: null, // No H2/H3 heading for this single chunk
+        original_document_path: relativePath,
+        start_line_in_doc: h1Found && lines.length > 0 && lines[0].trim().match(h1Regex) ? 2 : 1,
+        language: 'markdown',
+      });
+    }
+  }
+
+  if (!documentH1) {
+    documentH1 = path.basename(filePath).replace(path.extname(filePath), '');
+    console.log(`[extractMarkdownChunks DEBUG] H1 NOT FOUND for ${filePath}. Using fallback title: "${documentH1}"`);
+  }
+
+  return { chunks: chunks.filter((chunk) => chunk.content.length > 0), documentH1 };
+}

package/src/utils/mobilebert-tokenizer.js

@@ -0,0 +1,141 @@
+/**
+ * MobileBERT Tokenizer Utility
+ *
+ * Shared tokenizer functionality for MobileBERT models to handle token counting
+ * and text truncation while staying within the 512 token limit.
+ */
+
+import { AutoTokenizer } from '@huggingface/transformers';
+import chalk from 'chalk';
+
+// Shared tokenizer instance and initialization state
+let tokenizer = null;
+let isInitializing = false;
+let initializationPromise = null;
+
+/**
+ * Initialize and get the MobileBERT tokenizer (singleton pattern)
+ * @returns {Promise<AutoTokenizer|null>} Tokenizer instance or null if initialization fails
+ */
+async function getTokenizer() {
+  // If already initialized, return immediately
+  if (tokenizer) return tokenizer;
+
+  // If currently initializing, wait for the existing initialization
+  if (isInitializing && initializationPromise) {
+    return await initializationPromise;
+  }
+
+  // Start initialization
+  isInitializing = true;
+  initializationPromise = _initializeTokenizer();
+
+  try {
+    tokenizer = await initializationPromise;
+    return tokenizer;
+  } finally {
+    isInitializing = false;
+    initializationPromise = null;
+  }
+}
+
+/**
+ * Internal tokenizer initialization
+ * @returns {Promise<AutoTokenizer|null>}
+ */
+async function _initializeTokenizer() {
+  try {
+    console.log(chalk.blue('Initializing MobileBERT tokenizer...'));
+    const tok = await AutoTokenizer.from_pretrained('Xenova/mobilebert-uncased-mnli');
+    console.log(chalk.green('✓ MobileBERT tokenizer initialized successfully'));
+    return tok;
+  } catch (error) {
+    console.warn(chalk.yellow('⚠ Failed to initialize tokenizer, falling back to character estimation'), error.message);
+    return null;
+  }
+}
+
+/**
+ * Count exact tokens for MobileBERT model
+ * @param {string} text - Text to count tokens for
+ * @returns {Promise<number>} Number of tokens
+ */
+async function countTokens(text) {
+  if (!text || typeof text !== 'string') {
+    return 0;
+  }
+
+  try {
+    const tok = await getTokenizer();
+    if (!tok) {
+      // Fallback to character estimation if tokenizer fails
+      return Math.ceil(text.length / 3); // Conservative estimate for MobileBERT
+    }
+
+    const encoded = await tok.encode(text);
+    return encoded.length;
+  } catch (error) {
+    console.warn(chalk.gray('Token counting failed, using character estimation'), error.message);
+    return Math.ceil(text.length / 3);
+  }
+}
+
+/**
+ * Truncate text to fit within token limit while preserving important content
+ * @param {string} text - Text to truncate
+ * @param {number} maxTokens - Maximum tokens allowed (default: 450 for MobileBERT safety)
+ * @returns {Promise<string>} Truncated text
+ */
+export async function truncateToTokenLimit(text, maxTokens = 450) {
+  if (!text) return '';
+
+  const currentTokens = await countTokens(text);
+  if (currentTokens <= maxTokens) {
+    return text;
+  }
+
+  // Binary search to find the right length
+  let left = 0;
+  let right = text.length;
+  let bestLength = 0;
+
+  while (left <= right) {
+    const mid = Math.floor((left + right) / 2);
+    const truncated = text.substring(0, mid);
+    const tokens = await countTokens(truncated);
+
+    if (tokens <= maxTokens) {
+      bestLength = mid;
+      left = mid + 1;
+    } else {
+      right = mid - 1;
+    }
+  }
+
+  // Ensure we don't cut off in the middle of a word
+  let result = text.substring(0, bestLength);
+  const lastSpaceIndex = result.lastIndexOf(' ');
+  if (lastSpaceIndex > bestLength * 0.8) {
+    result = result.substring(0, lastSpaceIndex);
+  }
+
+  return result;
+}
+
+/**
+ * Clean up tokenizer resources
+ */
+export async function cleanupTokenizer() {
+  if (tokenizer) {
+    try {
+      if (typeof tokenizer.dispose === 'function') {
+        await tokenizer.dispose();
+      }
+      tokenizer = null;
+      console.log(chalk.green('✓ MobileBERT tokenizer resources cleaned up'));
+    } catch (error) {
+      console.warn(chalk.yellow('⚠ Error cleaning up tokenizer:'), error.message);
+      tokenizer = null;
+    }
+  }
+}