gitingest 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 472ca8e21d2bb9e48eeeada768679586b002087a9253f966eb164fd6118236d2
-  data.tar.gz: 1dddc890f5c380fb7ca2ca93db050473569dd1dd75c307f2d532857fc0f9ceaa
+  metadata.gz: fd7a1e5d5ced0b5449fa30671b0d9a536685d37c3d0d34d33437f652df24c199
+  data.tar.gz: c49a7c6489f7074e3870a05b5d1e47b0ea3e6b7a6eadce405db3c300d8165434
 SHA512:
-  metadata.gz: 1105676c81a389ea20a80a7be5ed2ea25c52fa2460c1672b0d5c45e19d2daf92c552a3d551cc71c196a9d0e2cbf96d779dc8e6b04f81965d323b31fff374f66a
-  data.tar.gz: 9b565545bfbfc1f983109cbdf08470b6399471f46dbcb651173ab5ab97a806b5ac992eb5affd1b58f71f2d71788d971aba9fde1aead94f779453636392253684
+  metadata.gz: 64b73ea01bc836a500c82a260c41be6f87e6f0c72bf868bee059407eea45466ad4892d905b7b69f5c6151e40e92d2553c8b8f678bfe1155fef968486648ae871
+  data.tar.gz: 7c18261e6fdb279916d2f8bff4557de76eb6ec0c039861d646f5f56dadacbef536b176d2a48fb25d142b70e30e868fb431f9cde6aad69262ee6f8dc37232ea0a

data/CHANGELOG.md

@@ -2,6 +2,27 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.3.0] - 2025-03-02
+- Added `faraday-retry` gem dependency for better API rate limit handling
+- Implemented thread-safe buffer management with mutex locks
+- Added new `ProgressIndicator` class for better CLI progress reporting (showing percentages)
+- Improved memory efficiency with configurable buffer size
+- Enhanced code organization with dedicated methods for file content formatting
+- Added comprehensive method documentation and parameter descriptions
+- Optimized thread pool size calculation for better performance
+- Improved error handling in concurrent operations
+
+## [0.2.0] - 2025-03-02
+- Added support for quiet and verbose modes in the command-line interface
+- Added the ability to specify a custom output file for the prompt
+- Enhanced error handling with logging support
+- Added logging functionality with custom loggers
+- Introduced rate limit handling with retries for file fetching
+- Added repository branch support
+- Exclude specific file patterns via command-line arguments
+- Enforced a 1000 file limit to prevent memory overload
+- Updated version to 0.2.0
+
 ## [0.1.0] - 2025-03-02
 
 ### Added
@@ -15,4 +36,4 @@ All notable changes to this project will be documented in this file.
 - Automatic rate limit handling with retry mechanism
 - Repository prompt generation with file separation markers
 - Support for custom branch selection
-- Custom output file naming options
+- Custom output file naming options

data/README.md

@@ -24,7 +24,26 @@ bundle exec rake install
 ### Command Line
 
 ```bash
-gitingest --repository username/repo --token YOUR_GITHUB_TOKEN --output output.txt
+# Basic usage (public repository)
+gitingest --repository user/repo 
+
+# With GitHub token for private repositories
+gitingest --repository user/repo --token YOUR_GITHUB_TOKEN
+
+# Specify a custom output file
+gitingest --repository user/repo --output my_prompt.txt
+
+# Specify a different branch
+gitingest --repository user/repo --branch develop
+
+# Exclude additional patterns
+gitingest --repository user/repo --exclude "*.md,docs/"
+
+# Quiet mode
+gitingest --repository user/repo --quiet
+
+# Verbose mode
+gitingest --repository user/repo --verbose
 ```
 
 #### Available Options
@@ -39,17 +58,32 @@ gitingest --repository username/repo --token YOUR_GITHUB_TOKEN --output output.t
 ### As a Library
 
 ```ruby
-require 'gitingest'
+require "gitingest"
 
-options = {
-  repository: 'davidesantangelo/gitingest',
-  token: 'your_github_token', # Optional
-  output_file: 'output.txt', # Optional
-  branch: 'master', # Optional
-  exclude: ['node_modules', '.*\.png$'] # Optional
-}
+# Basic usage
+generator = Gitingest::Generator.new(
+  repository: "user/repo",
+  token: "YOUR_GITHUB_TOKEN" # optional
+)
+generator.run
 
-generator = Gitingest::Generator.new(options)
+# With custom options
+generator = Gitingest::Generator.new(
+  repository: "user/repo",
+  token: "YOUR_GITHUB_TOKEN",
+  output_file: "my_prompt.txt",
+  branch: "develop",
+  exclude: ["*.md", "docs/"], 
+  quiet: true # or verbose: true
+)
+generator.run
+
+# With custom logger
+custom_logger = Logger.new("gitingest.log")
+generator = Gitingest::Generator.new(
+  repository: "user/repo",
+  logger: custom_logger
+)
 generator.run
 ```
 
@@ -64,19 +98,30 @@ generator.run
 
 ## Default Exclusion Patterns
 
-By default, Gitingest excludes the following files and directories:
+By default, the generator excludes files and directories commonly ignored in repositories, such as:
+
+- Version control files (`.git/`, `.svn/`)
+- System files (`.DS_Store`, `Thumbs.db`)
+- Log files (`*.log`, `*.bak`)
+- Images and media files (`*.png`, `*.jpg`, `*.mp3`)
+- Archives (`*.zip`, `*.tar.gz`)
+- Dependency directories (`node_modules/`, `vendor/`)
+- Compiled and binary files (`*.pyc`, `*.class`, `*.exe`)
 
-- `.git/`, `.github/`, `.gitignore`, `.DS_Store`
-- Log files: `.*\.log$`
-- Image files: `.*\.png$`, `.*\.jpg$`, `.*\.jpeg$`, `.*\.gif$`, `.*\.svg$`
-- Document files: `.*\.pdf$`
-- Archive files: `.*\.zip$`, `.*\.tar\.gz$`
-- Dependency directories: `node_modules/`, `vendor/`
+## Limitations
+
+- To prevent memory overload, only the first 1000 files will be processed
+- API requests are subject to GitHub limits (60 requests/hour without token, 5000 requests/hour with token)
+- Private repositories require a GitHub personal access token
 
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/davidesantangelo/gitingest.
 
+## Acknowledgements
+
+Inspired by [`cyclotruc/gitingest`](https://github.com/cyclotruc/gitingest).
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/bin/gitingest

@@ -6,15 +6,38 @@ require "gitingest"
 
 options = {}
 parser = OptionParser.new do |opts|
-  opts.banner = "Usage: gitingest [options]"
+  opts.banner = "Usage: gitingest [OPTIONS]"
+  opts.separator ""
+  opts.separator "Options:"
+
+  opts.on("-r", "--repository REPO", "GitHub repository (username/repo) [Required]") do |repo|
+    options[:repository] = repo
+  end
+
+  opts.on("-t", "--token TOKEN", "GitHub personal access token") do |token|
+    options[:token] = token
+  end
+
+  opts.on("-o", "--output FILE", "Output file for the prompt") do |file|
+    options[:output_file] = file
+  end
+
+  opts.on("-b", "--branch BRANCH", "Repository branch") do |branch|
+    options[:branch] = branch
+  end
 
-  opts.on("-r", "--repository REPO", "GitHub repository (username/repo)") { |repo| options[:repository] = repo }
-  opts.on("-t", "--token TOKEN", "GitHub personal access token") { |token| options[:token] = token }
-  opts.on("-o", "--output FILE", "Output file for the prompt") { |file| options[:output_file] = file }
   opts.on("-e", "--exclude PATTERN", "File patterns to exclude (comma separated)") do |pattern|
-    options[:exclude] = pattern.split(",").map(&:strip)
+    options[:exclude] = pattern.split(",")
   end
-  opts.on("-b", "--branch BRANCH", "Repository branch (default: main)") { |branch| options[:branch] = branch }
+
+  opts.on("-q", "--quiet", "Disable all output except errors") do
+    options[:quiet] = true
+  end
+
+  opts.on("-v", "--verbose", "Enable verbose output") do
+    options[:verbose] = true
+  end
+
   opts.on("-h", "--help", "Show this help message") do
     puts opts
     exit
@@ -23,9 +46,16 @@ end
 
 begin
   parser.parse!
+
+  if options[:repository].nil?
+    puts "Error: Repository option is required"
+    puts parser
+    exit 1
+  end
+
   generator = Gitingest::Generator.new(options)
   generator.run
-rescue ArgumentError => e
+rescue OptionParser::InvalidOption => e
   puts "Error: #{e.message}"
   puts parser
   exit 1

data/lib/gitingest/generator.rb

@@ -4,6 +4,7 @@ require "octokit"
 require "base64"
 require "fileutils"
 require "concurrent"
+require "logger"
 
 module Gitingest
   class Generator
@@ -66,19 +67,54 @@ module Gitingest
 
     # Maximum number of files to process to prevent memory overload
     MAX_FILES = 1000
-
-    attr_reader :options, :client, :repo_files, :excluded_patterns
-
+    BUFFER_SIZE = 100 # Write every 100 files to reduce I/O operations
+
+    attr_reader :options, :client, :repo_files, :excluded_patterns, :logger
+
+    # Initialize a new Generator with the given options
+    #
+    # @param options [Hash] Configuration options
+    # @option options [String] :repository GitHub repository in format "username/repo"
+    # @option options [String] :token GitHub personal access token
+    # @option options [String] :branch Repository branch (default: "main")
+    # @option options [String] :output_file Output file path
+    # @option options [Array<String>] :exclude Additional patterns to exclude
+    # @option options [Boolean] :quiet Reduce logging to errors only
+    # @option options [Boolean] :verbose Increase logging verbosity
+    # @option options [Logger] :logger Custom logger instance
     def initialize(options = {})
       @options = options
       @repo_files = []
       @excluded_patterns = []
+      setup_logger
       validate_options
       configure_client
       compile_excluded_patterns
     end
 
-    ### Option Validation
+    # Main execution method
+    def run
+      fetch_repository_contents
+      generate_prompt
+    end
+
+    private
+
+    # Set up logging based on verbosity options
+    def setup_logger
+      @logger = @options[:logger] || Logger.new($stdout)
+      @logger.level = if @options[:quiet]
+                        Logger::ERROR
+                      elsif @options[:verbose]
+                        Logger::DEBUG
+                      else
+                        Logger::INFO
+                      end
+      # Simplify logger format for command line usage
+      @logger.formatter = proc { |severity, _, _, msg| "#{severity == "INFO" ? "" : "[#{severity}] "}#{msg}\n" }
+    end
+
+    # Validate and set default options
     def validate_options
       raise ArgumentError, "Repository is required" unless @options[:repository]
 
@@ -88,37 +124,36 @@ module Gitingest
       @excluded_patterns = DEFAULT_EXCLUDES + @options[:exclude]
     end
 
-    ### Client Configuration
+    # Configure the GitHub API client
     def configure_client
       @client = @options[:token] ? Octokit::Client.new(access_token: @options[:token]) : Octokit::Client.new
 
       if @options[:token]
-        puts "Using provided GitHub token for authentication"
+        @logger.info "Using provided GitHub token for authentication"
       else
-        puts "Warning: No token provided. API rate limits will be restricted and private repositories will be inaccessible."
-        puts "For better results, provide a GitHub token with the --token option."
+        @logger.warn "Warning: No token provided. API rate limits will be restricted and private repositories will be inaccessible."
+        @logger.warn "For better results, provide a GitHub token with the --token option."
       end
     end
 
+    # Convert exclusion patterns to regular expressions
     def compile_excluded_patterns
       @excluded_patterns = @excluded_patterns.map { |pattern| Regexp.new(pattern) }
     end
 
-    ### Fetch Repository Contents
+    # Fetch repository contents and apply exclusion filters
     def fetch_repository_contents
-      puts "Fetching repository: #{@options[:repository]} (branch: #{@options[:branch]})"
+      @logger.info "Fetching repository: #{@options[:repository]} (branch: #{@options[:branch]})"
       begin
-        # First validate authentication and repository access
         validate_repository_access
-
         repo_tree = @client.tree(@options[:repository], @options[:branch], recursive: true)
         @repo_files = repo_tree.tree.select { |item| item.type == "blob" && !excluded_file?(item.path) }
 
         if @repo_files.size > MAX_FILES
-          puts "Warning: Found #{@repo_files.size} files, limited to #{MAX_FILES}."
+          @logger.warn "Warning: Found #{@repo_files.size} files, limited to #{MAX_FILES}."
           @repo_files = @repo_files.first(MAX_FILES)
         end
-        puts "Found #{@repo_files.size} files after exclusion filters"
+        @logger.info "Found #{@repo_files.size} files after exclusion filters"
       rescue Octokit::Unauthorized
         raise "Authentication error: Invalid or expired GitHub token. Please provide a valid token."
       rescue Octokit::NotFound
@@ -128,8 +163,8 @@ module Gitingest
       end
     end
 
+    # Validate repository and branch access
     def validate_repository_access
-      # Check if we can access the repository
       begin
         @client.repository(@options[:repository])
       rescue Octokit::Unauthorized
@@ -138,7 +173,6 @@ module Gitingest
         raise "Repository '#{@options[:repository]}' not found or is private. Check the repository name or provide a valid token."
       end
 
-      # Check if the branch exists
       begin
         @client.branch(@options[:repository], @options[:branch])
       rescue Octokit::NotFound
@@ -146,47 +180,64 @@ module Gitingest
       end
     end
 
+    # Check if a file should be excluded based on its path
     def excluded_file?(path)
       return true if path.start_with?(".") || path.split("/").any? { |part| part.start_with?(".") }
 
       @excluded_patterns.any? { |pattern| path.match?(pattern) }
     end
 
-    ### Generate Prompt
+    # Generate the consolidated prompt file
     def generate_prompt
-      puts "Generating prompt..."
-      Concurrent::Array.new(@repo_files)
+      @logger.info "Generating prompt..."
       buffer = []
-      buffer_size = 100 # Write every 100 files to reduce I/O
+      progress = ProgressIndicator.new(@repo_files.size, @logger)
 
       # Dynamic thread pool based on core count
-      pool = Concurrent::FixedThreadPool.new([Concurrent.processor_count, 5].max)
+      pool = Concurrent::FixedThreadPool.new([Concurrent.processor_count, 5].min)
 
       File.open(@options[:output_file], "w") do |file|
         @repo_files.each_with_index do |repo_file, index|
           pool.post do
             content = fetch_file_content_with_retry(repo_file.path)
-            result = <<~TEXT
-              ================================================================
-              File: #{repo_file.path}
-              ================================================================
-              #{content}
-
-            TEXT
-            buffer << result
-            write_buffer(file, buffer) if buffer.size >= buffer_size
-            print "\rProcessing: #{index + 1}/#{@repo_files.size} files"
+            result = format_file_content(repo_file.path, content)
+
+            # Thread-safe buffer management
+            buffer_mutex.synchronize do
+              buffer << result
+              write_buffer(file, buffer) if buffer.size >= BUFFER_SIZE
+            end
+
+            progress.update(index + 1)
           rescue Octokit::Error => e
-            puts "\nError fetching #{repo_file.path}: #{e.message}"
+            @logger.error "Error fetching #{repo_file.path}: #{e.message}"
           end
         end
+
         pool.shutdown
         pool.wait_for_termination
-        write_buffer(file, buffer) unless buffer.empty?
+
+        # Write any remaining files in buffer
+        buffer_mutex.synchronize do
+          write_buffer(file, buffer) unless buffer.empty?
+        end
       end
-      puts "\nPrompt generated and saved to #{@options[:output_file]}"
+
+      @logger.info "Prompt generated and saved to #{@options[:output_file]}"
     end
 
+    # Format a file's content for the prompt
+    def format_file_content(path, content)
+      <<~TEXT
+        ================================================================
+        File: #{path}
+        ================================================================
+        #{content}
+
+      TEXT
+    end
+
+    # Fetch file content with retry logic for rate limiting
     def fetch_file_content_with_retry(path, retries = 3)
       content = @client.contents(@options[:repository], path: path, ref: @options[:branch])
       Base64.decode64(content.content)
@@ -194,20 +245,37 @@ module Gitingest
       raise unless retries.positive?
 
       sleep_time = 60 / retries
-      puts "Rate limit exceeded, waiting #{sleep_time} seconds..."
+      @logger.warn "Rate limit exceeded, waiting #{sleep_time} seconds..."
       sleep(sleep_time)
       fetch_file_content_with_retry(path, retries - 1)
     end
 
+    # Write buffer contents to file and clear buffer
     def write_buffer(file, buffer)
       file.puts(buffer.join)
       buffer.clear
     end
 
-    ### Main Execution
-    def run
-      fetch_repository_contents
-      generate_prompt
+    # Thread-safe mutex for buffer operations
+    def buffer_mutex
+      @buffer_mutex ||= Mutex.new
+    end
+  end
+
+  # Helper class for showing progress in CLI
+  class ProgressIndicator
+    def initialize(total, logger)
+      @total = total
+      @logger = logger
+      @last_percent = 0
+    end
+
+    def update(current)
+      percent = (current.to_f / @total * 100).round
+      return unless percent > @last_percent && ((percent % 5).zero? || current == @total)
+
+      @logger.info "Processing: #{percent}% complete (#{current}/#{@total} files)"
+      @last_percent = percent
     end
   end
 end

data/lib/gitingest/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Gitingest
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitingest
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Davide Santangelo
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: faraday-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: octokit
   requirement: !ruby/object:Gem::Requirement