codesummary 1.2.1 → 1.2.2

package/src/scanner.js

@@ -1,6 +1,7 @@
 import fs from 'fs-extra';
 import path from 'path';
 import chalk from 'chalk';
+import ignore from 'ignore';
 import ErrorHandler from './errorHandler.js';
 import { formatFileSize, getExtensionDescription, matchesGlobPattern } from './utils.js';
 
@@ -14,6 +15,7 @@ export class Scanner {
     this.allowedExtensions = new Set(config.allowedExtensions.map(ext => ext.toLowerCase()));
     this.excludeDirs = new Set(config.excludeDirs);
     this.excludeFiles = config.excludeFiles || [];
+    this.csIgnoreMatchers = new Map();
   }
 
   /**
@@ -40,6 +42,8 @@ export class Scanner {
       }
 
       console.log(chalk.gray(`Scanning directory: ${resolvedRoot}`));
+      this.csIgnoreMatchers.clear();
+      await this.preloadCsIgnoreFiles(resolvedRoot, scanWarnings);
 
       const filesByExtension = {};
       const scannedFiles = new Set(); // Prevent duplicates
@@ -51,7 +55,7 @@ export class Scanner {
         processedFiles: 0
       };
 
-      await this.walkDirectory(resolvedRoot, resolvedRoot, filesByExtension, scannedFiles, scanContext);
+      await this.walkDirectory(resolvedRoot, resolvedRoot, filesByExtension, scannedFiles, scanContext, []);
 
       // Sort files within each extension group
       Object.keys(filesByExtension).forEach(ext => {
@@ -80,8 +84,9 @@ export class Scanner {
    * @param {Set} scannedFiles - Set to track processed files and avoid duplicates
    * @param {object} scanContext - Context object to track scan statistics
    */
-  async walkDirectory(currentPath, rootPath, filesByExtension, scannedFiles, scanContext) {
+  async walkDirectory(currentPath, rootPath, filesByExtension, scannedFiles, scanContext, inheritedIgnoreChain = []) {
     try {
+      const ignoreChain = await this.buildIgnoreChainForDirectory(currentPath, inheritedIgnoreChain);
       const entries = await fs.readdir(currentPath, { withFileTypes: true });
 
       for (const entry of entries) {
@@ -90,16 +95,16 @@ export class Scanner {
 
         if (entry.isDirectory()) {
           // Skip excluded directories and hidden directories (unless explicitly allowed)
-          if (this.shouldSkipDirectory(entry.name, relativePath)) {
+          if (this.shouldSkipDirectory(entry.name, relativePath, fullPath, ignoreChain)) {
             scanContext.skippedDirectories++;
             continue;
           }
 
           // Recursively scan subdirectory
-          await this.walkDirectory(fullPath, rootPath, filesByExtension, scannedFiles, scanContext);
+          await this.walkDirectory(fullPath, rootPath, filesByExtension, scannedFiles, scanContext, ignoreChain);
         } else if (entry.isFile()) {
           // Process file if it matches criteria
-          await this.processFile(fullPath, rootPath, filesByExtension, scannedFiles, scanContext);
+          await this.processFile(fullPath, rootPath, filesByExtension, scannedFiles, scanContext, ignoreChain);
         } else if (entry.isSymbolicLink()) {
           // Handle symbolic links with caution
           scanContext.warnings.push(`Skipped symbolic link: ${relativePath}`);
@@ -130,7 +135,7 @@ export class Scanner {
    * @param {Set} scannedFiles - Set of already processed files
    * @param {object} scanContext - Context object to track scan statistics
    */
-  async processFile(fullPath, rootPath, filesByExtension, scannedFiles, scanContext) {
+  async processFile(fullPath, rootPath, filesByExtension, scannedFiles, scanContext, ignoreChain = []) {
     try {
       const relativePath = path.relative(rootPath, fullPath);
       
@@ -140,6 +145,11 @@ export class Scanner {
       }
       scannedFiles.add(fullPath);
 
+      if (this.shouldSkipByCsIgnore(fullPath, false, ignoreChain)) {
+        scanContext.skippedFiles++;
+        return;
+      }
+
       const extension = path.extname(relativePath).toLowerCase();
 
       // Skip files without extensions or not in allowed list
@@ -215,7 +225,11 @@ export class Scanner {
    * @param {string} relativePath - Relative path from root
    * @returns {boolean} True if directory should be skipped
    */
-  shouldSkipDirectory(dirName, relativePath) {
+  shouldSkipDirectory(dirName, relativePath, fullPath, ignoreChain = []) {
+    if (this.shouldSkipByCsIgnore(fullPath, true, ignoreChain)) {
+      return true;
+    }
+
     // Skip directories in exclude list
     if (this.excludeDirs.has(dirName)) {
       return true;
@@ -239,6 +253,99 @@ export class Scanner {
     return false;
   }
 
+  /**
+   * Preload every .csignore file under scan root.
+   * @param {string} rootPath
+   * @param {Array<string>} warnings
+   */
+  async preloadCsIgnoreFiles(rootPath, warnings = []) {
+    const stack = [rootPath];
+
+    while (stack.length > 0) {
+      const current = stack.pop();
+      if (!current) continue;
+
+      let entries = [];
+      try {
+        entries = await fs.readdir(current, { withFileTypes: true });
+      } catch {
+        continue;
+      }
+
+      for (const entry of entries) {
+        const fullPath = path.join(current, entry.name);
+        if (entry.isDirectory()) {
+          if (this.excludeDirs.has(entry.name)) continue;
+          if (entry.name.startsWith('.') && !this.isAllowedHiddenDirectory(entry.name)) continue;
+          stack.push(fullPath);
+          continue;
+        }
+
+        if (!entry.isFile() || entry.name !== '.csignore') continue;
+
+        try {
+          const content = await fs.readFile(fullPath, 'utf8');
+          const matcher = ignore();
+          matcher.add(content);
+          const negationMatcher = ignore();
+          const negationPatterns = content
+            .split(/\r\n|\r|\n/)
+            .map(line => line.trim())
+            .filter(line => line.startsWith('!') && line.length > 1)
+            .map(line => line.slice(1).trim())
+            .filter(line => line.length > 0 && !line.startsWith('#'));
+          if (negationPatterns.length > 0) {
+            negationMatcher.add(negationPatterns);
+          }
+
+          this.csIgnoreMatchers.set(current, { matcher, negationMatcher, hasNegations: negationPatterns.length > 0 });
+          console.log(chalk.gray(`Using .csignore rules from ${fullPath}`));
+        } catch (error) {
+          warnings.push(`Failed to read .csignore (${fullPath}): ${error.message}`);
+        }
+      }
+    }
+  }
+
+  /**
+   * Build ignore chain from scan root to current directory.
+   * @param {string} directoryPath
+   * @param {Array<object>} inheritedIgnoreChain
+   * @returns {Promise<Array<object>>}
+   */
+  async buildIgnoreChainForDirectory(directoryPath, inheritedIgnoreChain = []) {
+    const layer = this.csIgnoreMatchers.get(directoryPath);
+    if (!layer) return inheritedIgnoreChain;
+    return [...inheritedIgnoreChain, { baseDir: directoryPath, ...layer }];
+  }
+
+  /**
+   * Check whether a path is ignored by .csignore rules using gitignore-like
+   * precedence across ancestor directories.
+   * @param {string} absolutePath - Absolute path to file or directory
+   * @param {boolean} isDirectory - Whether path is a directory
+   * @param {Array<object>} ignoreChain - Ordered list of ancestor ignore matchers
+   * @returns {boolean}
+   */
+  shouldSkipByCsIgnore(absolutePath, isDirectory = false, ignoreChain = []) {
+    if (!ignoreChain || ignoreChain.length === 0) return false;
+    let ignored = false;
+
+    for (const layer of ignoreChain) {
+      const relative = path.relative(layer.baseDir, absolutePath).replace(/\\/g, '/');
+      if (!relative || relative.startsWith('..')) continue;
+      const candidate = isDirectory ? `${relative}/` : relative;
+      const result = layer.matcher.test(candidate);
+      if (result.ignored) ignored = true;
+      if (result.unignored) ignored = false;
+      if (!result.ignored && !result.unignored && layer.hasNegations && layer.negationMatcher.ignores(candidate)) {
+        ignored = false;
+      }
+    }
+
+    return ignored;
+  }
+
   /**
    * Check if a file should be excluded based on patterns
    * @param {string} fileName - File name to check
@@ -384,4 +491,4 @@ export class Scanner {
   }
 }
 
-export default Scanner;
+export default Scanner;

package/rag-schema.json

@@ -1,114 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "$id": "https://github.com/skamoll/CodeSummary/schemas/rag-output.json",
-  "title": "CodeSummary RAG Output Schema",
-  "description": "Schema for CodeSummary RAG-optimized JSON output format",
-  "type": "object",
-  "required": ["metadata", "files", "index"],
-  "properties": {
-    "metadata": {
-      "type": "object",
-      "required": ["projectName", "generatedAt", "version", "generator"],
-      "properties": {
-        "projectName": { "type": "string" },
-        "generatedAt": { "type": "string", "format": "date-time" },
-        "version": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" },
-        "generator": { "type": "string" },
-        "scanPath": { "type": "string" },
-        "config": { "type": "object" },
-        "summary": { "type": "object" }
-      }
-    },
-    "files": {
-      "type": "array",
-      "items": {
-        "type": "object",
-        "required": ["id", "path", "extension", "language", "hash", "chunks"],
-        "properties": {
-          "id": { "type": "string" },
-          "path": { "type": "string" },
-          "extension": { "type": "string" },
-          "language": { "type": "string" },
-          "size": { "type": "number", "minimum": 0 },
-          "hash": { "type": "string", "pattern": "^sha256-[a-f0-9]{64}$" },
-          "modified": { "type": "string", "format": "date-time" },
-          "tags": { "type": "array", "items": { "type": "string" } },
-          "chunks": {
-            "type": "array",
-            "items": {
-              "type": "object",
-              "required": ["id", "content", "tokenEstimate"],
-              "properties": {
-                "id": { "type": "string" },
-                "content": { "type": "string" },
-                "tokenEstimate": { "type": "number", "minimum": 0 },
-                "lineStart": { "type": "number", "minimum": 1 },
-                "lineEnd": { "type": "number", "minimum": 1 },
-                "chunkingMethod": { "type": "string" },
-                "imports": { "type": "array", "items": { "type": "string" } },
-                "calls": { "type": "array", "items": { "type": "string" } }
-              }
-            }
-          }
-        }
-      }
-    },
-    "index": {
-      "type": "object",
-      "required": ["version", "generatedAt", "summary", "chunkOffsets", "fileOffsets", "statistics"],
-      "properties": {
-        "version": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" },
-        "generatedAt": { "type": "string", "format": "date-time" },
-        "summary": {
-          "type": "object",
-          "required": ["fileCount", "chunkCount", "totalBytes"],
-          "properties": {
-            "fileCount": { "type": "number", "minimum": 0 },
-            "chunkCount": { "type": "number", "minimum": 0 },
-            "totalBytes": { "type": "number", "minimum": 0 },
-            "languages": { "type": "array", "items": { "type": "string" } },
-            "extensions": { "type": "array", "items": { "type": "string" } }
-          }
-        },
-        "chunkOffsets": {
-          "type": "object",
-          "patternProperties": {
-            "^chunk_[a-f0-9]+_\\d+$": {
-              "type": "object",
-              "required": ["jsonStart", "jsonEnd", "contentStart", "contentEnd", "filePath"],
-              "properties": {
-                "jsonStart": { "type": "number", "minimum": 0 },
-                "jsonEnd": { "type": "number", "minimum": 0 },
-                "contentStart": { "type": "number", "minimum": 0 },
-                "contentEnd": { "type": "number", "minimum": 0 },
-                "filePath": { "type": "string" }
-              }
-            }
-          }
-        },
-        "fileOffsets": {
-          "type": "object",
-          "patternProperties": {
-            "^[a-f0-9]+$": {
-              "type": "array",
-              "items": { "type": "number", "minimum": 0 },
-              "minItems": 2,
-              "maxItems": 2
-            }
-          }
-        },
-        "statistics": {
-          "type": "object",
-          "required": ["processingTimeMs", "bytesWritten", "totalFiles"],
-          "properties": {
-            "processingTimeMs": { "type": "number", "minimum": 1 },
-            "bytesPerSecond": { "type": "number", "minimum": 0 },
-            "bytesWritten": { "type": "number", "minimum": 0 },
-            "totalFiles": { "type": "number", "minimum": 0 },
-            "emptyFiles": { "type": "number", "minimum": 0 }
-          }
-        }
-      }
-    }
-  }
-}

package/src/ragConfig.js

@@ -1,369 +0,0 @@
-import fs from "fs-extra";
-import yaml from "js-yaml";
-import path from "path";
-import { matchesGlobPattern } from "./utils.js";
-
-/**
- * RAG Configuration Manager
- * Loads and validates configuration from raggen.config.yaml
- */
-export class RagConfigManager {
-  constructor() {
-    this.defaultConfig = this.getDefaultConfig();
-    this.configPath = null;
-    this.loadedConfig = null;
-  }
-
-  /**
-   * Load configuration from YAML file
-   * @param {string} configPath - Path to config file (optional)
-   * @returns {object} Merged configuration
-   */
-  async loadConfig(configPath = null) {
-    // Try to find config file
-    this.configPath = configPath || (await this.findConfigFile());
-
-    if (this.configPath && (await fs.pathExists(this.configPath))) {
-      try {
-        const yamlContent = await fs.readFile(this.configPath, "utf8");
-        const userConfig = yaml.load(yamlContent);
-
-        // Merge with defaults
-        this.loadedConfig = this.mergeConfigs(this.defaultConfig, userConfig);
-
-        console.log(`📋 RAG config loaded from: ${this.configPath}`);
-        return this.loadedConfig;
-      } catch (error) {
-        console.warn(`⚠️  Error loading RAG config: ${error.message}`);
-        console.log(`📋 Using default RAG configuration`);
-        return this.defaultConfig;
-      }
-    } else {
-      console.log(`📋 No RAG config found, using defaults`);
-      return this.defaultConfig;
-    }
-  }
-
-  /**
-   * Find configuration file in common locations
-   * @returns {string|null} Path to config file or null
-   */
-  async findConfigFile() {
-    const searchPaths = [
-      "raggen.config.yaml",
-      "raggen.config.yml",
-      ".raggen.config.yaml",
-      ".raggen.config.yml",
-      "config/raggen.yaml",
-      "config/raggen.yml",
-    ];
-
-    for (const searchPath of searchPaths) {
-      if (await fs.pathExists(searchPath)) {
-        return path.resolve(searchPath);
-      }
-    }
-
-    return null;
-  }
-
-  /**
-   * Get default configuration
-   * @returns {object} Default config
-   */
-  getDefaultConfig() {
-    return {
-      extensions: {
-        include: [
-          ".json",
-          ".ts",
-          ".js",
-          ".jsx",
-          ".tsx",
-          ".xml",
-          ".html",
-          ".css",
-          ".scss",
-          ".md",
-          ".txt",
-          ".py",
-          ".java",
-          ".cs",
-          ".cpp",
-          ".c",
-          ".h",
-          ".yaml",
-          ".yml",
-          ".sh",
-          ".bat",
-        ],
-      },
-      chunking: {
-        maxTokens: 1000,
-        overlap: 200,
-        tokenEstimation: "ceil(length/4)",
-      },
-      handlers: {
-        code: {
-          splitByFunction: true,
-          detectImports: true,
-          detectCalls: true,
-          complexityAnalysis: true,
-        },
-        markup: {
-          splitByElement: true,
-          preserveStructure: true,
-        },
-        styling: {
-          splitByRule: true,
-          detectImports: true,
-        },
-        config: {
-          splitBySection: true,
-          validateSyntax: false,
-        },
-      },
-      paths: {
-        exclude: [
-          "node_modules",
-          ".git",
-          "dist",
-          "build",
-          "coverage",
-          "out",
-          "__pycache__",
-          ".next",
-          ".nuxt",
-          ".cache",
-          "tmp",
-          "temp",
-          "logs",
-          "bower_components",
-          "vendor",
-        ],
-      },
-      files: {
-        exclude: [
-          "*-lock.json",
-          "*.lock",
-          "composer.lock",
-          "Pipfile.lock",
-          "*.min.js",
-          "*.min.css",
-          "*.map",
-          ".DS_Store",
-          "Thumbs.db",
-          "*-lock.yaml",
-        ],
-      },
-      performance: {
-        maxWorkers: 1,
-        batchSize: 50,
-        maxFileSize: "100MB",
-        streamingThreshold: "10MB",
-      },
-      output: {
-        format: "json",
-        compression: false,
-        validation: true,
-        indexing: true,
-      },
-      metadata: {
-        calculateHashes: true,
-        extractTags: true,
-        trackRelationships: true,
-        includeStats: true,
-      },
-      logging: {
-        level: "info",
-        progressReporting: true,
-        statisticsReporting: true,
-      },
-      quality: {
-        maxChunkSize: "50KB",
-        maxOutputSize: "250MB",
-        duplicateDetection: true,
-        emptyChunkHandling: "skip",
-      },
-    };
-  }
-
-  /**
-   * Deep merge configuration objects
-   * @param {object} defaultConfig - Default configuration
-   * @param {object} userConfig - User configuration
-   * @returns {object} Merged configuration
-   */
-  mergeConfigs(defaultConfig, userConfig) {
-    const merged = JSON.parse(JSON.stringify(defaultConfig)); // Deep clone
-
-    return this.deepMerge(merged, userConfig);
-  }
-
-  /**
-   * Recursively merge objects
-   * @param {object} target - Target object
-   * @param {object} source - Source object
-   * @returns {object} Merged object
-   */
-  deepMerge(target, source) {
-    for (const key in source) {
-      if (source.hasOwnProperty(key)) {
-        if (
-          source[key] &&
-          typeof source[key] === "object" &&
-          !Array.isArray(source[key])
-        ) {
-          // Recursive merge for objects
-          if (!target[key] || typeof target[key] !== "object") {
-            target[key] = {};
-          }
-          this.deepMerge(target[key], source[key]);
-        } else {
-          // Direct assignment for primitives and arrays
-          target[key] = source[key];
-        }
-      }
-    }
-    return target;
-  }
-
-  /**
-   * Validate configuration
-   * @param {object} config - Configuration to validate
-   * @returns {boolean} True if valid
-   */
-  validateConfig(config) {
-    const errors = [];
-
-    // Validate required sections
-    const requiredSections = ["extensions", "chunking", "handlers"];
-    for (const section of requiredSections) {
-      if (!config[section]) {
-        errors.push(`Missing required section: ${section}`);
-      }
-    }
-
-    // Validate chunking settings
-    if (config.chunking) {
-      if (
-        typeof config.chunking.maxTokens !== "number" ||
-        config.chunking.maxTokens <= 0
-      ) {
-        errors.push("chunking.maxTokens must be a positive number");
-      }
-      if (
-        typeof config.chunking.overlap !== "number" ||
-        config.chunking.overlap < 0
-      ) {
-        errors.push("chunking.overlap must be a non-negative number");
-      }
-    }
-
-    // Validate extensions
-    if (config.extensions && config.extensions.include) {
-      if (!Array.isArray(config.extensions.include)) {
-        errors.push("extensions.include must be an array");
-      } else {
-        for (const ext of config.extensions.include) {
-          if (typeof ext !== "string" || !ext.startsWith(".")) {
-            errors.push(`Invalid extension: ${ext} (must start with dot)`);
-          }
-        }
-      }
-    }
-
-    if (errors.length > 0) {
-      console.error("❌ RAG Configuration validation errors:");
-      errors.forEach((error) => console.error(`   • ${error}`));
-      return false;
-    }
-
-    return true;
-  }
-
-  /**
-   * Get configuration value with dot notation
-   * @param {string} path - Configuration path (e.g., 'chunking.maxTokens')
-   * @param {any} defaultValue - Default value if not found
-   * @returns {any} Configuration value
-   */
-  get(path, defaultValue = null) {
-    const config = this.loadedConfig || this.defaultConfig;
-
-    return path.split(".").reduce((obj, key) => {
-      return obj && obj[key] !== undefined ? obj[key] : defaultValue;
-    }, config);
-  }
-
-  /**
-   * Display current configuration
-   */
-  displayConfig() {
-    const config = this.loadedConfig || this.defaultConfig;
-
-    console.log("\n📋 RAG Generator Configuration:");
-    console.log(`   Source: ${this.configPath ? this.configPath : "defaults"}`);
-    console.log(`   Extensions: ${config.extensions.include.length} types`);
-    console.log(`   Max tokens per chunk: ${config.chunking.maxTokens}`);
-    console.log(`   Token overlap: ${config.chunking.overlap}`);
-    console.log(`   Max workers: ${config.performance.maxWorkers}`);
-    console.log(`   Batch size: ${config.performance.batchSize}`);
-    console.log();
-  }
-
-  /**
-   * Parse file size string to bytes
-   * @param {string} sizeStr - Size string (e.g., '100MB', '1GB')
-   * @returns {number} Size in bytes
-   */
-  parseFileSize(sizeStr) {
-    if (typeof sizeStr === "number") return sizeStr;
-
-    const units = {
-      B: 1,
-      KB: 1024,
-      MB: 1024 * 1024,
-      GB: 1024 * 1024 * 1024,
-    };
-
-    const match = sizeStr.match(/^(\d+(?:\.\d+)?)\s*([KMGT]?B)$/i);
-    if (!match) return 0;
-
-    const value = parseFloat(match[1]);
-    const unit = match[2].toUpperCase();
-
-    return Math.floor(value * (units[unit] || 1));
-  }
-
-  /**
-   * Check if file should be excluded by path
-   * @param {string} filePath - File path to check
-   * @returns {boolean} True if should be excluded
-   */
-  shouldExcludePath(filePath) {
-    const config = this.loadedConfig || this.defaultConfig;
-    const excludePaths = config.paths?.exclude || [];
-
-    return excludePaths.some((pattern) => {
-      return (
-        filePath.includes(pattern) ||
-        filePath.includes(path.sep + pattern + path.sep)
-      );
-    });
-  }
-
-  /**
-   * Check if file should be excluded by filename pattern
-   * @param {string} fileName - File name to check
-   * @returns {boolean} True if should be excluded
-   */
-  shouldExcludeFile(fileName) {
-    const config = this.loadedConfig || this.defaultConfig;
-    const excludeFiles = config.files?.exclude || [];
-
-    return excludeFiles.some((pattern) => matchesGlobPattern(fileName, pattern));
-  }
-}
-
-export default RagConfigManager;