omnizip 0.4.7-x86_64-linux

data/lib/omnizip/parallel/parallel_extractor.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Omnizip
+  module Parallel
+    # Parallel extraction coordinator using Fractor
+    #
+    # Manages parallel extraction of files from an archive.
+    # Distributes extraction work across multiple workers and
+    # writes files to disk in a thread-safe manner.
+    #
+    # @example Extract archive in parallel
+    #   extractor = Omnizip::Parallel::ParallelExtractor.new(threads: 4)
+    #   extractor.extract('backup.zip', 'output/')
+    #
+    # @example With options
+    #   options = Omnizip::Models::ParallelOptions.new
+    #   options.threads = 8
+    #   extractor = Omnizip::Parallel::ParallelExtractor.new(options)
+    #   extractor.extract('backup.zip', 'output/')
+    class ParallelExtractor
+      # @return [Omnizip::Models::ParallelOptions] parallel options
+      attr_reader :options
+
+      # @return [Hash] extraction statistics
+      attr_reader :stats
+
+      # Initialize parallel extractor
+      #
+      # @param options [Omnizip::Models::ParallelOptions, Hash] parallel options
+      # @param threads [Integer] number of threads (overrides options)
+      def initialize(options = nil, threads: nil)
+        @options = case options
+                   when Omnizip::Models::ParallelOptions
+                     options.dup
+                   when Hash
+                     Omnizip::Models::ParallelOptions.new.apply(options)
+                   else
+                     Omnizip::Models::ParallelOptions.new
+                   end
+
+        @options.threads = threads if threads
+        @options.validate_options!
+
+        @stats = {
+          files_extracted: 0,
+          bytes_extracted: 0,
+          start_time: nil,
+          end_time: nil,
+        }
+
+        @write_mutex = Mutex.new
+      end
+
+      # Extract archive to directory in parallel
+      #
+      # @param archive [String] archive path
+      # @param dest [String] destination directory
+      # @param options [Hash] extraction options
+      # @option options [Boolean] :overwrite overwrite existing files
+      # @option options [Proc] :progress progress callback
+      # @return [Array<String>] extracted file paths
+      def extract(archive, dest, **options)
+        unless ::File.exist?(archive)
+          raise Errno::ENOENT,
+                "Archive not found: #{archive}"
+        end
+
+        overwrite = options.fetch(:overwrite, false)
+        options[:progress]
+
+        @stats[:start_time] = Time.now
+
+        # Read archive to get entries
+        entries = read_archive_entries(archive)
+
+        # Create destination directory
+        FileUtils.mkdir_p(dest)
+
+        # Create job queue
+        job_queue = JobQueue.new(max_size: @options.queue_size)
+
+        # Schedule jobs
+        entries.each do |entry|
+          job_queue.push_with_size(
+            file: entry.filename,
+            size: safe_entry_size(entry),
+            data: { entry_name: entry.filename },
+          )
+        end
+
+        # Create work items from jobs, largest first
+        work_items = []
+        until job_queue.empty?
+          job = job_queue.pop(timeout: 0.1)
+          break unless job
+
+          work_items << job.data[:entry_name]
+        end
+
+        # Threads, not Ractors: Omnizip's autoloads and Proc-holding
+        # codec constants cannot cross Ractor boundaries on Ruby <= 3.3
+        pool = ThreadPool.new(size: @options.threads)
+        raw_results, raw_errors = pool.map(work_items) do |entry_name|
+          process_entry(archive, dest, entry_name)
+        end
+
+        # Handle errors (raw_errors holds nil for every item that
+        # succeeded — zip+compact would keep those pairs)
+        failure_pairs = raw_errors.each_with_index.filter_map do |error, index|
+          [work_items[index], error] if error
+        end
+        unless failure_pairs.empty?
+          error_msgs = failure_pairs.map { |name, e| "#{name}: #{e.message}" }
+            .join("\n")
+          raise Omnizip::DecompressionError, "Extraction errors:\n#{error_msgs}"
+        end
+
+        results = raw_results
+
+        # Write files to disk (thread-safe)
+        extracted_paths = write_extracted_files(results, overwrite: overwrite)
+
+        @stats[:end_time] = Time.now
+        @stats[:files_extracted] = results.size
+
+        extracted_paths
+      end
+
+      # Get extraction statistics
+      #
+      # @return [Hash] statistics
+      def statistics
+        duration = if @stats[:start_time] && @stats[:end_time]
+                     @stats[:end_time] - @stats[:start_time]
+                   else
+                     0
+                   end
+
+        @stats.merge(
+          duration: duration,
+          throughput_mbps: calculate_throughput(duration),
+        )
+      end
+
+      private
+
+      # Decompress one entry in a worker thread. Each worker opens
+      # the archive independently — its own file handle per thread.
+      def process_entry(archive_path, dest_dir, entry_name)
+        entry = Omnizip::Formats::Zip::Reader.new(archive_path)
+          .entries.find { |e| e.filename == entry_name }
+        raise Errno::ENOENT, "Entry not found: #{entry_name}" unless entry
+
+        {
+          entry_name: entry_name,
+          dest_path: ::File.join(dest_dir, entry_name),
+          data: entry.directory? ? "" : read_entry_data(archive_path, entry_name),
+          directory: entry.directory?,
+          unix_perms: entry.unix_permissions.to_i,
+        }
+      end
+
+      def read_entry_data(archive_path, entry_name)
+        Omnizip::Formats::Zip::Reader.new(archive_path)
+          .read_entry(entry_name)
+      end
+
+      def safe_entry_size(entry)
+        entry.uncompressed_size.to_i
+      end
+
+      # Read archive entries through the native reader
+      #
+      # @param archive_path [String] archive path
+      # @return [Array<CentralDirectoryHeader>] array of entries
+      def read_archive_entries(archive_path)
+        Omnizip::Formats::Zip::Reader.new(archive_path).entries
+      end
+
+      # Write extracted files to disk
+      #
+      # @param results [Array] extraction results
+      # @param overwrite [Boolean] overwrite existing files
+      # @return [Array<String>] extracted file paths
+      def write_extracted_files(results, overwrite: false)
+        extracted_paths = []
+
+        results.each do |result|
+          next unless result
+
+          dest_path = result[:dest_path]
+
+          # Thread-safe file writing
+          @write_mutex.synchronize do
+            if result[:directory]
+              # mkdir_p is idempotent — a file entry may already have
+              # created this directory as its parent
+              FileUtils.mkdir_p(dest_path)
+              extracted_paths << dest_path
+              next
+            end
+
+            # Check if file exists
+            if ::File.exist?(dest_path) && !overwrite
+              raise Errno::EEXIST, "File exists: #{dest_path}"
+            end
+
+            FileUtils.mkdir_p(::File.dirname(dest_path))
+            ::File.binwrite(dest_path, result[:data])
+
+            # Set permissions if Unix
+            if result[:unix_perms].positive?
+              ::File.chmod(result[:unix_perms] & 0o777, dest_path)
+            end
+
+            @stats[:bytes_extracted] += result[:data].bytesize
+
+            extracted_paths << dest_path
+          end
+        end
+
+        extracted_paths
+      end
+
+      # Calculate throughput in MB/s
+      #
+      # @param duration [Float] duration in seconds
+      # @return [Float] throughput in MB/s
+      def calculate_throughput(duration)
+        return 0.0 if duration.zero?
+
+        (@stats[:bytes_extracted].to_f / (1024 * 1024)) / duration
+      end
+    end
+  end
+end

data/lib/omnizip/parallel/thread_pool.rb

@@ -0,0 +1,60 @@
+# frozen_string: true
+
+module Omnizip
+  module Parallel
+    # Bounded thread pool for library work.
+    #
+    # Unlike the Fractor pool, threads share the VM state — Ractor
+    # workers cannot trigger Omnizip's autoloads or touch Proc-holding
+    # codec constants on Ruby <= 3.3 ("require by autoload on
+    # non-main Ractor is not supported"). Decompression itself still
+    # parallelizes: the zlib/bzip2 C extensions release the GVL.
+    class ThreadPool
+      # @return [Integer] number of worker threads
+      attr_reader :size
+
+      # Initialize the pool
+      #
+      # @param size [Integer] worker thread count (at least 1)
+      def initialize(size:)
+        @size = [size.to_i, 1].max
+      end
+
+      # Run +block+ for every item, at most +size+ at a time, and
+      # return a pair of parallel arrays: results (nil where an item
+      # raised) and errors (nil where it succeeded). Item order is
+      # preserved in both arrays.
+      #
+      # @param items [Array] work items
+      # @yieldparam item [Object] one work item
+      # @return [Array<Array, Array>]
+      def map(items)
+        results = Array.new(items.size)
+        errors = Array.new(items.size)
+        queue = Queue.new
+        items.each_with_index { |item, index| queue << [item, index] }
+
+        threads = Array.new(@size) do
+          Thread.new do
+            loop do
+              begin
+                item, index = queue.pop(true)
+              rescue ThreadError
+                break
+              end
+
+              begin
+                results[index] = yield(item)
+              rescue StandardError => e
+                errors[index] = e
+              end
+            end
+          end
+        end
+        threads.each(&:join)
+
+        [results, errors]
+      end
+    end
+  end
+end

data/lib/omnizip/parallel/worker_pool.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require "fractor"
+
+module Omnizip
+  module Parallel
+    # Worker pool wrapper for Fractor-based parallel processing
+    #
+    # Manages a pool of Fractor workers for parallel compression/extraction.
+    # Handles job distribution, result collection, and graceful shutdown.
+    #
+    # @example Create and use worker pool
+    #   pool = Omnizip::Parallel::WorkerPool.new(
+    #     worker_class: CompressionWorker,
+    #     num_workers: 4
+    #   )
+    #   pool.start
+    #   pool.submit(work_item)
+    #   results = pool.results
+    #   pool.shutdown
+    class WorkerPool
+      # @return [Fractor::Supervisor] underlying Fractor supervisor
+      attr_reader :supervisor
+
+      # @return [Array] collected results
+      attr_reader :results
+
+      # @return [Array] collected errors
+      attr_reader :errors
+
+      # @return [Boolean] whether pool is running
+      attr_reader :running
+
+      # Initialize worker pool
+      #
+      # @param worker_class [Class] Fractor::Worker subclass
+      # @param num_workers [Integer] number of worker threads
+      # @param continuous [Boolean] continuous mode for long-running tasks
+      def initialize(worker_class:, num_workers: nil, continuous: false)
+        @worker_class = worker_class
+        @num_workers = num_workers || detect_cpu_count
+        @continuous = continuous
+        @supervisor = nil
+        @results = []
+        @errors = []
+        @running = false
+        @result_mutex = Mutex.new
+        @work_queue = nil
+      end
+
+      # Start the worker pool
+      #
+      # @return [void]
+      def start
+        return if @running
+
+        # Create Fractor supervisor with worker pool configuration
+        @supervisor = Fractor::Supervisor.new(
+          worker_pools: [
+            {
+              worker_class: @worker_class,
+              num_workers: @num_workers,
+            },
+          ],
+          continuous_mode: @continuous,
+        )
+
+        # For continuous mode, set up work queue
+        if @continuous
+          @work_queue = Fractor::WorkQueue.new
+          @work_queue.register_with_supervisor(@supervisor)
+        end
+
+        @running = true
+
+        # Start supervisor in background thread for continuous mode
+        if @continuous
+          @supervisor_thread = Thread.new do
+            @supervisor.run
+          rescue StandardError => e
+            @result_mutex.synchronize do
+              @errors << { error: e, message: "Supervisor error: #{e.message}" }
+            end
+          end
+        else
+          # For batch mode, don't start yet - wait for work items
+          @supervisor.start_workers
+        end
+      end
+
+      # Submit work item to the pool
+      #
+      # @param work [Fractor::Work] work item to process
+      # @return [void]
+      def submit(work)
+        raise Omnizip::IOError, "Worker pool not started" unless @running
+
+        if @continuous
+          # In continuous mode, add to work queue
+          @work_queue << work
+        else
+          # In batch mode, add to supervisor
+          @supervisor.add_work_item(work)
+        end
+      end
+
+      # Submit multiple work items
+      #
+      # @param works [Array<Fractor::Work>] array of work items
+      # @return [void]
+      def submit_batch(works)
+        works.each { |work| submit(work) }
+      end
+
+      # Run the pool in batch mode and wait for completion
+      #
+      # @return [void]
+      def run
+        raise Omnizip::IOError, "Can only run in batch mode" if @continuous
+        raise Omnizip::IOError, "Worker pool not started" unless @running
+
+        @supervisor.run
+
+        # Collect results
+        collect_results
+      end
+
+      # Shutdown the worker pool
+      #
+      # @param timeout [Numeric] timeout in seconds
+      # @return [void]
+      def shutdown(timeout: 30)
+        return unless @running
+
+        # Stop in BOTH modes: batch worker Ractors otherwise linger
+        # in Ractor.receive for the life of the process, burning CPU
+        # after the batch completes
+        @supervisor&.stop
+
+        if @continuous
+          @supervisor_thread&.join(timeout)
+        end
+
+        # Collect final results
+        collect_results
+
+        @running = false
+      end
+
+      # Get successful results
+      #
+      # @return [Array] array of successful results
+      def successful_results
+        @result_mutex.synchronize { @results.dup }
+      end
+
+      # Get failed results
+      #
+      # @return [Array] array of errors
+      def failed_results
+        @result_mutex.synchronize { @errors.dup }
+      end
+
+      # Get pool statistics
+      #
+      # @return [Hash] statistics hash
+      def stats
+        return {} unless @supervisor
+
+        {
+          workers: @num_workers,
+          running: @running,
+          continuous: @continuous,
+          results: @results.size,
+          errors: @errors.size,
+          total_processed: @results.size + @errors.size,
+        }
+      end
+
+      # Check if pool has completed all work
+      #
+      # @return [Boolean] true if complete
+      def complete?
+        return false if @continuous
+        return false unless @supervisor
+
+        # In batch mode, check if all work is processed
+        result_aggregator = @supervisor.results
+        result_aggregator && !@supervisor.work_queue.empty?
+      end
+
+      private
+
+      # Collect results from supervisor
+      #
+      # @return [void]
+      def collect_results
+        return unless @supervisor
+
+        result_aggregator = @supervisor.results
+        return unless result_aggregator
+
+        @result_mutex.synchronize do
+          # Collect successful results
+          result_aggregator.results.each do |work_result|
+            @results << work_result
+          end
+
+          # Collect errors
+          result_aggregator.errors.each do |error_result|
+            @errors << error_result
+          end
+        end
+      end
+
+      # Detect number of available CPU cores
+      #
+      # @return [Integer] number of CPUs
+      def detect_cpu_count
+        require "etc"
+        Etc.nprocessors
+      rescue StandardError
+        4 # fallback
+      end
+    end
+  end
+end

data/lib/omnizip/parallel.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module Omnizip
+  # Parallel processing module for multi-threaded compression/extraction
+  #
+  # Leverages Fractor for parallel processing to utilize multi-core CPUs.
+  # Provides high-level APIs for parallel compression and extraction operations.
+  #
+  # @example Auto-detect CPU count and compress in parallel
+  #   Omnizip::Parallel.compress_directory('files/', 'backup.zip')
+  #
+  # @example Custom thread count
+  #   Omnizip::Parallel.compress_directory('files/', 'backup.zip', threads: 8)
+  #
+  # @example Parallel extraction
+  #   Omnizip::Parallel.extract_archive('large.zip', 'output/', threads: 4)
+  #
+  # @example Configure globally
+  #   Omnizip::Parallel.configure do |config|
+  #     config.default_threads = 8
+  #     config.queue_size = 100
+  #     config.load_balancing = :dynamic
+  #   end
+  module Parallel
+    # Nested classes - autoloaded
+    autoload :JobQueue, "omnizip/parallel/job_queue"
+    autoload :JobScheduler, "omnizip/parallel/job_scheduler"
+    autoload :WorkerPool, "omnizip/parallel/worker_pool"
+    autoload :Engine, "omnizip/parallel/engine"
+    autoload :ParallelCompressor, "omnizip/parallel/parallel_compressor"
+    autoload :ParallelExtractor, "omnizip/parallel/parallel_extractor"
+    autoload :ThreadPool, "omnizip/parallel/thread_pool"
+
+    class << self
+      # Global configuration
+      attr_accessor :config
+
+      # Configure parallel processing globally
+      #
+      # @yield [config] Configuration block
+      # @yieldparam config [Omnizip::Models::ParallelOptions] configuration object
+      # @return [void]
+      #
+      # @example
+      #   Omnizip::Parallel.configure do |config|
+      #     config.threads = 8
+      #     config.queue_size = 100
+      #     config.strategy = :dynamic
+      #   end
+      def configure
+        @config ||= Omnizip::Models::ParallelOptions.new
+        yield @config if block_given?
+        @config.validate_options!
+        @config
+      end
+
+      # Compress directory in parallel
+      #
+      # @param dir [String] directory path
+      # @param output [String] output archive path
+      # @param options [Hash] compression options
+      # @option options [Integer] :threads number of threads
+      # @option options [Symbol] :compression compression method
+      # @option options [Integer] :level compression level
+      # @option options [Boolean] :recursive include subdirectories
+      # @option options [Proc] :progress progress callback
+      # @return [String] path to created archive
+      #
+      # @example
+      #   Omnizip::Parallel.compress_directory('files/', 'backup.zip')
+      #   Omnizip::Parallel.compress_directory('files/', 'backup.zip', threads: 8)
+      def compress_directory(dir, output, **options)
+        threads = options.delete(:threads) || @config&.threads
+
+        compressor = ParallelCompressor.new(@config, threads: threads)
+        result = compressor.compress(dir, output, **options)
+
+        # Print stats if verbose
+        if options[:verbose] || @config&.verbose
+          stats = compressor.statistics
+          print_stats("Compression", stats)
+        end
+
+        result
+      end
+
+      # Extract archive in parallel
+      #
+      # @param archive [String] archive path
+      # @param dest [String] destination directory
+      # @param options [Hash] extraction options
+      # @option options [Integer] :threads number of threads
+      # @option options [Boolean] :overwrite overwrite existing files
+      # @option options [Proc] :progress progress callback
+      # @return [Array<String>] extracted file paths
+      #
+      # @example
+      #   Omnizip::Parallel.extract_archive('large.zip', 'output/')
+      #   Omnizip::Parallel.extract_archive('large.zip', 'output/', threads: 4)
+      def extract_archive(archive, dest, **options)
+        threads = options.delete(:threads) || @config&.threads
+
+        extractor = ParallelExtractor.new(@config, threads: threads)
+        result = extractor.extract(archive, dest, **options)
+
+        # Print stats if verbose
+        if options[:verbose] || @config&.verbose
+          stats = extractor.statistics
+          print_stats("Extraction", stats)
+        end
+
+        result
+      end
+
+      # Get default configuration
+      #
+      # @return [Omnizip::Models::ParallelOptions] configuration
+      def default_config
+        @default_config ||= Omnizip::Models::ParallelOptions.new
+      end
+
+      # Reset configuration to defaults
+      #
+      # @return [void]
+      def reset_config
+        @config = Omnizip::Models::ParallelOptions.new
+      end
+
+      private
+
+      # Print statistics
+      #
+      # @param operation [String] operation name
+      # @param stats [Hash] statistics hash
+      def print_stats(operation, stats)
+        puts "\n=== #{operation} Statistics ==="
+        puts "Files processed: #{stats[:files_processed] || stats[:files_extracted] || 0}"
+        puts "Duration: #{'%.2f' % stats[:duration]}s"
+        puts "Throughput: #{'%.2f' % stats[:throughput_mbps]} MB/s"
+
+        if stats[:compression_ratio]
+          puts "Compression ratio: #{'%.2f' % stats[:compression_ratio]}%"
+        end
+
+        puts "================================\n"
+      end
+    end
+
+    # Initialize default configuration
+    def self.setup_default_config
+      @config = Omnizip::Models::ParallelOptions.new
+    end
+
+    setup_default_config
+  end
+end