gitingest 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14bcb35132327c7725e69a895d56b4e88fb21db78fa5473c9afb4d08b879b7ee
-  data.tar.gz: f3a5e06bec7566a268342678887bac14989d69d26f10cbb590e1f594c6779b89
+  metadata.gz: 114b6d8e954843f88917f556760e7fab89753126cb5fd059ed6cfe55c8f2e6c6
+  data.tar.gz: 39873c0d14eff9d6e615f6e2f13e7e30331f0e97db1162b613047782a582ff6c
 SHA512:
-  metadata.gz: 1aed7d97acae8b6a1c2b15757cdc9852d802c99b83aa73caf114dfc904d0b05c44b70c853cd453fc12868310c09b15bf6bf71dcac203c01b5dfde241dc7ab0f5
-  data.tar.gz: 07145ca986675723ad0371c873c176384631da660bed227b31ef4cf2680d72cc98479b4da8fb990fedb3dce0eab74ca5114c1222eea0ca1890a85d73726a8d4b
+  metadata.gz: f35ffb41ddd9990cd1c2e8c4ad08018a5bdd63c8ec222bfce021ede2c67dfa9355fbc7f1102f014c555d0df46501877b01f2767aadf674896635bd7fa5bab355
+  data.tar.gz: e431693ee45cf65ab3c63931c5c4a69675ddafc8038f90cda3dfb7dacec7449f48a65a41bc12f56996f8e697f80d1a896243e97aa37c96085107491099dabc05

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## [0.4.0] - 2025-03-03
+
+### Added
+- Added `generate_prompt` method for in-memory content generation without file I/O
+- Integrated visual progress bar with file processing rate reporting
+- Added human-readable time formatting for progress estimates
+- Enhanced test coverage for multithreaded operations
+
+### Changed
+- Refactored `process_content_to_output` for better code reuse between file and string output
+- Improved thread management to handle various error conditions more gracefully
+- Enhanced documentation with programmatic usage examples
+
+### Fixed
+- Resolved thread pool shutdown issues in test environment
+- Fixed race conditions in progress indicator updates
+- Addressed timing inconsistencies in multithreaded test scenarios
+
+---
+
 ## [0.3.1] - 2025-03-03
 
 ### Added

data/README.md

@@ -39,6 +39,15 @@ gitingest --repository user/repo --branch develop
 # Exclude additional patterns
 gitingest --repository user/repo --exclude "*.md,docs/"
 
+# Control the number of threads
+gitingest --repository user/repo -T 4
+
+# Set thread pool shutdown timeout
+gitingest --repository user/repo -W 120
+
+# Combine threading options
+gitingest --repository user/repo -T 8 -W 90
+
 # Quiet mode
 gitingest --repository user/repo --quiet
 
@@ -53,6 +62,10 @@ gitingest --repository user/repo --verbose
 - `-o, --output FILE`: Output file for the prompt [Default: reponame_prompt.txt]
 - `-e, --exclude PATTERN`: File patterns to exclude (comma separated)
 - `-b, --branch BRANCH`: Repository branch [Default: main]
+- `-T, --threads COUNT`: Number of concurrent threads [Default: auto-detected]
+- `-W, --thread-timeout SECONDS`: Thread pool shutdown timeout [Default: 60]
+- `-q, --quiet`: Reduce logging to errors only
+- `-v, --verbose`: Increase logging verbosity
 - `-h, --help`: Show help message
 
 ### As a Library
@@ -60,13 +73,21 @@ gitingest --repository user/repo --verbose
 ```ruby
 require "gitingest"
 
-# Basic usage
+# Basic usage - write to a file
 generator = Gitingest::Generator.new(
   repository: "user/repo",
   token: "YOUR_GITHUB_TOKEN" # optional
 )
+
+# Run the full workflow (fetch repository and generate file)
 generator.run
 
+# OR generate file only (if you need the output path)
+output_path = generator.generate_file
+
+# Get content as a string (for in-memory processing)
+content = generator.generate_prompt
+
 # With custom options
 generator = Gitingest::Generator.new(
   repository: "user/repo",
@@ -74,9 +95,10 @@ generator = Gitingest::Generator.new(
   output_file: "my_prompt.txt",
   branch: "develop",
   exclude: ["*.md", "docs/"], 
-  quiet: true # or verbose: true
+  threads: 4,              # control concurrency
+  thread_timeout: 120,     # custom thread timeout
+  quiet: true              # or verbose: true
 )
-generator.run
 
 # With custom logger
 custom_logger = Logger.new("gitingest.log")
@@ -84,7 +106,6 @@ generator = Gitingest::Generator.new(
   repository: "user/repo",
   logger: custom_logger
 )
-generator.run
 ```
 
 ## Features

data/bin/gitingest

@@ -22,7 +22,7 @@ parser = OptionParser.new do |opts|
     options[:output_file] = file
   end
 
-  opts.on("-b", "--branch BRANCH", "Repository branch") do |branch|
+  opts.on("-b", "--branch BRANCH", "Repository branch [Default: main]") do |branch|
     options[:branch] = branch
   end
 
@@ -30,11 +30,20 @@ parser = OptionParser.new do |opts|
     options[:exclude] = pattern.split(",")
   end
 
-  opts.on("-q", "--quiet", "Disable all output except errors") do
+  opts.on("-T", "--threads COUNT", Integer, "Number of concurrent threads [Default: auto-detected]") do |threads|
+    options[:threads] = threads
+  end
+
+  opts.on("-W", "--thread-timeout SECONDS", Integer,
+          "Thread pool shutdown timeout in seconds [Default: 60]") do |timeout|
+    options[:thread_timeout] = timeout
+  end
+
+  opts.on("-q", "--quiet", "Reduce logging to errors only") do
     options[:quiet] = true
   end
 
-  opts.on("-v", "--verbose", "Enable verbose output") do
+  opts.on("-v", "--verbose", "Increase logging verbosity") do
     options[:verbose] = true
   end
 
@@ -42,6 +51,13 @@ parser = OptionParser.new do |opts|
     puts opts
     exit
   end
+
+  opts.separator ""
+  opts.separator "Examples:"
+  opts.separator "  gitingest -r user/repo                      # Basic usage with public repository"
+  opts.separator "  gitingest -r user/repo -t YOUR_TOKEN        # With GitHub token for private repositories"
+  opts.separator "  gitingest -r user/repo -o custom_prompt.txt # Custom output file"
+  opts.separator "  gitingest -r user/repo -T 8                 # Use 8 threads for processing"
 end
 
 begin

data/lib/gitingest/generator.rb

@@ -106,10 +106,42 @@ module Gitingest
       compile_excluded_patterns
     end
 
-    # Main execution method
+    # Main execution method for command line
     def run
       fetch_repository_contents
-      generate_prompt
+      generate_file
+    end
+
+    # Generate content and save it to a file
+    #
+    # @return [String] Path to the generated file
+    def generate_file
+      fetch_repository_contents if @repo_files.empty?
+
+      @logger.info "Generating file for #{@options[:repository]}"
+      File.open(@options[:output_file], "w") do |file|
+        process_content_to_output(file)
+      end
+
+      @logger.info "Prompt generated and saved to #{@options[:output_file]}"
+      @options[:output_file]
+    end
+
+    # Generate content and return it as a string
+    # Useful for programmatic usage
+    #
+    # @return [String] The generated repository content
+    def generate_prompt
+      @logger.info "Generating in-memory prompt for #{@options[:repository]}"
+
+      fetch_repository_contents if @repo_files.empty?
+
+      content = StringIO.new
+      process_content_to_output(content)
+
+      result = content.string
+      @logger.info "Generated #{result.size} bytes of content in memory"
+      result
     end
 
     private
@@ -202,89 +234,82 @@ module Gitingest
       path.match?(@combined_exclude_regex)
     end
 
-    # Generate the consolidated prompt file with optimized threading
-    def generate_prompt
-      @logger.info "Generating prompt..."
+    # Common implementation for both file and string output
+    def process_content_to_output(output)
       @logger.debug "Using thread pool with #{@options[:threads]} threads"
 
       buffer = []
       progress = ProgressIndicator.new(@repo_files.size, @logger)
 
-      # Optimization: thread-local buffers to reduce mutex contention
+      # Thread-local buffers to reduce mutex contention
       thread_buffers = {}
       mutex = Mutex.new
       errors = []
 
-      # Dynamic thread pool based on configuration
+      # Thread pool based on configuration
       pool = Concurrent::FixedThreadPool.new(@options[:threads])
 
-      # Group files by priority (smaller files first for better parallelism)
+      # Group files by priority
       prioritized_files = prioritize_files(@repo_files)
 
-      File.open(@options[:output_file], "w") do |file|
-        prioritized_files.each_with_index do |repo_file, index|
-          pool.post do
-            # Optimization: Use thread-local buffers
-            thread_id = Thread.current.object_id
-            thread_buffers[thread_id] ||= []
-            local_buffer = thread_buffers[thread_id]
-
-            begin
-              content = fetch_file_content_with_retry(repo_file.path)
-              result = format_file_content(repo_file.path, content)
-              local_buffer << result
-
-              # Optimization: Only acquire mutex when local buffer reaches threshold
-              if local_buffer.size >= LOCAL_BUFFER_THRESHOLD
-                mutex.synchronize do
-                  buffer.concat(local_buffer)
-                  write_buffer(file, buffer) if buffer.size >= BUFFER_SIZE
-                  local_buffer.clear
-                end
-              end
+      prioritized_files.each_with_index do |repo_file, index|
+        pool.post do
+          thread_id = Thread.current.object_id
+          thread_buffers[thread_id] ||= []
+          local_buffer = thread_buffers[thread_id]
 
-              progress.update(index + 1)
-            rescue Octokit::Error => e
-              mutex.synchronize do
-                errors << "Error fetching #{repo_file.path}: #{e.message}"
-                @logger.error "Error fetching #{repo_file.path}: #{e.message}"
-              end
-            rescue StandardError => e
+          begin
+            content = fetch_file_content_with_retry(repo_file.path)
+            result = format_file_content(repo_file.path, content)
+            local_buffer << result
+
+            if local_buffer.size >= LOCAL_BUFFER_THRESHOLD
               mutex.synchronize do
-                errors << "Unexpected error processing #{repo_file.path}: #{e.message}"
-                @logger.error "Unexpected error processing #{repo_file.path}: #{e.message}"
+                buffer.concat(local_buffer)
+                write_buffer(output, buffer) if buffer.size >= BUFFER_SIZE
+                local_buffer.clear
               end
             end
+
+            progress.update(index + 1)
+          rescue Octokit::Error => e
+            mutex.synchronize do
+              errors << "Error fetching #{repo_file.path}: #{e.message}"
+              @logger.error "Error fetching #{repo_file.path}: #{e.message}"
+            end
+          rescue StandardError => e
+            mutex.synchronize do
+              errors << "Unexpected error processing #{repo_file.path}: #{e.message}"
+              @logger.error "Unexpected error processing #{repo_file.path}: #{e.message}"
+            end
           end
         end
+      end
 
-        begin
-          pool.shutdown
-          wait_success = pool.wait_for_termination(@options[:thread_timeout])
+      begin
+        pool.shutdown
+        wait_success = pool.wait_for_termination(@options[:thread_timeout])
 
-          unless wait_success
-            @logger.warn "Thread pool did not shut down within #{@options[:thread_timeout]} seconds, forcing termination"
-            pool.kill
-          end
-        rescue StandardError => e
-          @logger.error "Error during thread pool shutdown: #{e.message}"
+        unless wait_success
+          @logger.warn "Thread pool did not shut down within #{@options[:thread_timeout]} seconds, forcing termination"
+          pool.kill
         end
+      rescue StandardError => e
+        @logger.error "Error during thread pool shutdown: #{e.message}"
+      end
 
-        # Process remaining files in thread-local buffers
-        mutex.synchronize do
-          thread_buffers.each_value do |local_buffer|
-            buffer.concat(local_buffer) unless local_buffer.empty?
-          end
-          write_buffer(file, buffer) unless buffer.empty?
+      # Process remaining files in thread-local buffers
+      mutex.synchronize do
+        thread_buffers.each_value do |local_buffer|
+          buffer.concat(local_buffer) unless local_buffer.empty?
         end
+        write_buffer(output, buffer) unless buffer.empty?
       end
 
-      if errors.any?
-        @logger.warn "Completed with #{errors.size} errors"
-        @logger.debug "First few errors: #{errors.first(3).join(", ")}" if @logger.debug?
-      end
+      return unless errors.any?
 
-      @logger.info "Prompt generated and saved to #{@options[:output_file]}"
+      @logger.warn "Completed with #{errors.size} errors"
+      @logger.debug "First few errors: #{errors.first(3).join(", ")}" if @logger.debug?
     end
 
     # Format a file's content for the prompt

data/lib/gitingest/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Gitingest
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitingest
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Davide Santangelo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-03-02 00:00:00.000000000 Z
+date: 2025-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby