omnizip 0.3.1

data/lib/omnizip/parallel/worker_pool.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require "fractor"
+
+module Omnizip
+  module Parallel
+    # Worker pool wrapper for Fractor-based parallel processing
+    #
+    # Manages a pool of Fractor workers for parallel compression/extraction.
+    # Handles job distribution, result collection, and graceful shutdown.
+    #
+    # @example Create and use worker pool
+    #   pool = Omnizip::Parallel::WorkerPool.new(
+    #     worker_class: CompressionWorker,
+    #     num_workers: 4
+    #   )
+    #   pool.start
+    #   pool.submit(work_item)
+    #   results = pool.results
+    #   pool.shutdown
+    class WorkerPool
+      # @return [Fractor::Supervisor] underlying Fractor supervisor
+      attr_reader :supervisor
+
+      # @return [Array] collected results
+      attr_reader :results
+
+      # @return [Array] collected errors
+      attr_reader :errors
+
+      # @return [Boolean] whether pool is running
+      attr_reader :running
+
+      # Initialize worker pool
+      #
+      # @param worker_class [Class] Fractor::Worker subclass
+      # @param num_workers [Integer] number of worker threads
+      # @param continuous [Boolean] continuous mode for long-running tasks
+      def initialize(worker_class:, num_workers: nil, continuous: false)
+        @worker_class = worker_class
+        @num_workers = num_workers || detect_cpu_count
+        @continuous = continuous
+        @supervisor = nil
+        @results = []
+        @errors = []
+        @running = false
+        @result_mutex = Mutex.new
+        @work_queue = nil
+      end
+
+      # Start the worker pool
+      #
+      # @return [void]
+      def start
+        return if @running
+
+        # Create Fractor supervisor with worker pool configuration
+        @supervisor = Fractor::Supervisor.new(
+          worker_pools: [
+            {
+              worker_class: @worker_class,
+              num_workers: @num_workers,
+            },
+          ],
+          continuous_mode: @continuous,
+        )
+
+        # For continuous mode, set up work queue
+        if @continuous
+          @work_queue = Fractor::WorkQueue.new
+          @work_queue.register_with_supervisor(@supervisor)
+        end
+
+        @running = true
+
+        # Start supervisor in background thread for continuous mode
+        if @continuous
+          @supervisor_thread = Thread.new do
+            @supervisor.run
+          rescue StandardError => e
+            @result_mutex.synchronize do
+              @errors << { error: e, message: "Supervisor error: #{e.message}" }
+            end
+          end
+        else
+          # For batch mode, don't start yet - wait for work items
+          @supervisor.start_workers
+        end
+      end
+
+      # Submit work item to the pool
+      #
+      # @param work [Fractor::Work] work item to process
+      # @return [void]
+      def submit(work)
+        raise "Worker pool not started" unless @running
+
+        if @continuous
+          # In continuous mode, add to work queue
+          @work_queue << work
+        else
+          # In batch mode, add to supervisor
+          @supervisor.add_work_item(work)
+        end
+      end
+
+      # Submit multiple work items
+      #
+      # @param works [Array<Fractor::Work>] array of work items
+      # @return [void]
+      def submit_batch(works)
+        works.each { |work| submit(work) }
+      end
+
+      # Run the pool in batch mode and wait for completion
+      #
+      # @return [void]
+      def run
+        raise "Can only run in batch mode" if @continuous
+        raise "Worker pool not started" unless @running
+
+        @supervisor.run
+
+        # Collect results
+        collect_results
+      end
+
+      # Shutdown the worker pool
+      #
+      # @param timeout [Numeric] timeout in seconds
+      # @return [void]
+      def shutdown(timeout: 30)
+        return unless @running
+
+        if @continuous
+          @supervisor.stop
+          @supervisor_thread&.join(timeout)
+        end
+
+        # Collect final results
+        collect_results
+
+        @running = false
+      end
+
+      # Get successful results
+      #
+      # @return [Array] array of successful results
+      def successful_results
+        @result_mutex.synchronize { @results.dup }
+      end
+
+      # Get failed results
+      #
+      # @return [Array] array of errors
+      def failed_results
+        @result_mutex.synchronize { @errors.dup }
+      end
+
+      # Get pool statistics
+      #
+      # @return [Hash] statistics hash
+      def stats
+        return {} unless @supervisor
+
+        {
+          workers: @num_workers,
+          running: @running,
+          continuous: @continuous,
+          results: @results.size,
+          errors: @errors.size,
+          total_processed: @results.size + @errors.size,
+        }
+      end
+
+      # Check if pool has completed all work
+      #
+      # @return [Boolean] true if complete
+      def complete?
+        return false if @continuous
+        return false unless @supervisor
+
+        # In batch mode, check if all work is processed
+        result_aggregator = @supervisor.results
+        result_aggregator && !@supervisor.work_queue.empty?
+      end
+
+      private
+
+      # Collect results from supervisor
+      #
+      # @return [void]
+      def collect_results
+        return unless @supervisor
+
+        result_aggregator = @supervisor.results
+        return unless result_aggregator
+
+        @result_mutex.synchronize do
+          # Collect successful results
+          result_aggregator.results.each do |work_result|
+            @results << work_result
+          end
+
+          # Collect errors
+          result_aggregator.errors.each do |error_result|
+            @errors << error_result
+          end
+        end
+      end
+
+      # Detect number of available CPU cores
+      #
+      # @return [Integer] number of CPUs
+      def detect_cpu_count
+        require "etc"
+        Etc.nprocessors
+      rescue StandardError
+        4 # fallback
+      end
+    end
+  end
+end

data/lib/omnizip/parallel.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require_relative "parallel/job_queue"
+require_relative "parallel/job_scheduler"
+require_relative "parallel/worker_pool"
+require_relative "parallel/parallel_compressor"
+require_relative "parallel/parallel_extractor"
+
+module Omnizip
+  # Parallel processing module for multi-threaded compression/extraction
+  #
+  # Leverages Fractor for parallel processing to utilize multi-core CPUs.
+  # Provides high-level APIs for parallel compression and extraction operations.
+  #
+  # @example Auto-detect CPU count and compress in parallel
+  #   Omnizip::Parallel.compress_directory('files/', 'backup.zip')
+  #
+  # @example Custom thread count
+  #   Omnizip::Parallel.compress_directory('files/', 'backup.zip', threads: 8)
+  #
+  # @example Parallel extraction
+  #   Omnizip::Parallel.extract_archive('large.zip', 'output/', threads: 4)
+  #
+  # @example Configure globally
+  #   Omnizip::Parallel.configure do |config|
+  #     config.default_threads = 8
+  #     config.queue_size = 100
+  #     config.load_balancing = :dynamic
+  #   end
+  module Parallel
+    class << self
+      # Global configuration
+      attr_accessor :config
+
+      # Configure parallel processing globally
+      #
+      # @yield [config] Configuration block
+      # @yieldparam config [Omnizip::Models::ParallelOptions] configuration object
+      # @return [void]
+      #
+      # @example
+      #   Omnizip::Parallel.configure do |config|
+      #     config.threads = 8
+      #     config.queue_size = 100
+      #     config.strategy = :dynamic
+      #   end
+      def configure
+        @config ||= Omnizip::Models::ParallelOptions.new
+        yield @config if block_given?
+        @config.validate!
+        @config
+      end
+
+      # Compress directory in parallel
+      #
+      # @param dir [String] directory path
+      # @param output [String] output archive path
+      # @param options [Hash] compression options
+      # @option options [Integer] :threads number of threads
+      # @option options [Symbol] :compression compression method
+      # @option options [Integer] :level compression level
+      # @option options [Boolean] :recursive include subdirectories
+      # @option options [Proc] :progress progress callback
+      # @return [String] path to created archive
+      #
+      # @example
+      #   Omnizip::Parallel.compress_directory('files/', 'backup.zip')
+      #   Omnizip::Parallel.compress_directory('files/', 'backup.zip', threads: 8)
+      def compress_directory(dir, output, **options)
+        threads = options.delete(:threads) || @config&.threads
+
+        compressor = ParallelCompressor.new(@config, threads: threads)
+        result = compressor.compress(dir, output, **options)
+
+        # Print stats if verbose
+        if options[:verbose] || @config&.verbose
+          stats = compressor.statistics
+          print_stats("Compression", stats)
+        end
+
+        result
+      end
+
+      # Extract archive in parallel
+      #
+      # @param archive [String] archive path
+      # @param dest [String] destination directory
+      # @param options [Hash] extraction options
+      # @option options [Integer] :threads number of threads
+      # @option options [Boolean] :overwrite overwrite existing files
+      # @option options [Proc] :progress progress callback
+      # @return [Array<String>] extracted file paths
+      #
+      # @example
+      #   Omnizip::Parallel.extract_archive('large.zip', 'output/')
+      #   Omnizip::Parallel.extract_archive('large.zip', 'output/', threads: 4)
+      def extract_archive(archive, dest, **options)
+        threads = options.delete(:threads) || @config&.threads
+
+        extractor = ParallelExtractor.new(@config, threads: threads)
+        result = extractor.extract(archive, dest, **options)
+
+        # Print stats if verbose
+        if options[:verbose] || @config&.verbose
+          stats = extractor.statistics
+          print_stats("Extraction", stats)
+        end
+
+        result
+      end
+
+      # Get default configuration
+      #
+      # @return [Omnizip::Models::ParallelOptions] configuration
+      def default_config
+        @default_config ||= Omnizip::Models::ParallelOptions.new
+      end
+
+      # Reset configuration to defaults
+      #
+      # @return [void]
+      def reset_config
+        @config = Omnizip::Models::ParallelOptions.new
+      end
+
+      private
+
+      # Print statistics
+      #
+      # @param operation [String] operation name
+      # @param stats [Hash] statistics hash
+      def print_stats(operation, stats)
+        puts "\n=== #{operation} Statistics ==="
+        puts "Files processed: #{stats[:files_processed] || stats[:files_extracted] || 0}"
+        puts "Duration: #{'%.2f' % stats[:duration]}s"
+        puts "Throughput: #{'%.2f' % stats[:throughput_mbps]} MB/s"
+
+        if stats[:compression_ratio]
+          puts "Compression ratio: #{'%.2f' % stats[:compression_ratio]}%"
+        end
+
+        puts "================================\n"
+      end
+    end
+
+    # Initialize default configuration
+    configure {}
+  end
+end

data/lib/omnizip/parity/chunked_block_processor.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+require_relative "reed_solomon_matrix"
+
+module Omnizip
+  module Parity
+    # Chunked block processor for incremental Reed-Solomon recovery
+    #
+    # Implements correct RS decoding: x = A^-1 * b
+    # where b[i] = recovery[i] - sum(present[k] * base[present[k]]^exponent[i])
+    #
+    # Processes large blocks incrementally in memory-efficient chunks.
+    class ChunkedBlockProcessor
+      # Default chunk size (1MB)
+      DEFAULT_CHUNK_SIZE = 1024 * 1024
+
+      # @return [ReedSolomonMatrix] RS matrix with precomputed coefficients
+      attr_reader :matrix
+
+      # @return [Hash<Integer, String>] Present input blocks (index => data)
+      attr_reader :present_blocks
+
+      # @return [Hash<Integer, String>] Recovery blocks (exponent => data)
+      attr_reader :recovery_blocks
+
+      # @return [Array<Integer>] Missing block indices to recover
+      attr_reader :missing_indices
+
+      # @return [Integer] Block size in bytes
+      attr_reader :block_size
+
+      # @return [Integer] Chunk size for processing
+      attr_reader :chunk_size
+
+      # Initialize processor
+      #
+      # @param matrix [ReedSolomonMatrix] Precomputed RS matrix (A^-1)
+      # @param present_blocks [Hash<Integer, String>] Present data blocks
+      # @param recovery_blocks [Hash<Integer, String>] Recovery blocks (by exponent)
+      # @param missing_indices [Array<Integer>] Indices to recover
+      # @param block_size [Integer] Block size in bytes
+      # @param chunk_size [Integer] Chunk size for processing
+      def initialize(matrix, present_blocks, recovery_blocks, missing_indices,
+ block_size, chunk_size: DEFAULT_CHUNK_SIZE)
+        @matrix = matrix
+        @present_blocks = present_blocks
+        @recovery_blocks = recovery_blocks
+        @missing_indices = missing_indices.sort
+        @block_size = block_size
+        # Ensure chunk_size is even (we process 16-bit words)
+        requested_chunk = [chunk_size, block_size].min
+        @chunk_size = requested_chunk - (requested_chunk % 2)
+      end
+
+      # Process all blocks incrementally
+      #
+      # Implements: x = A^-1 * b
+      # where b = recovery - present_contributions
+      #
+      # @return [Hash<Integer, String>] Recovered blocks
+      def process_all
+        # Initialize output blocks (all zeros)
+        recovered = {}
+        missing_indices.each do |idx|
+          recovered[idx] = "\x00".b * block_size
+        end
+
+        # Process block chunk by chunk
+        block_offset = 0
+        while block_offset < block_size
+          current_chunk_size = [chunk_size, block_size - block_offset].min
+          process_chunk_at_offset(recovered, block_offset, current_chunk_size)
+          block_offset += current_chunk_size
+        end
+
+        recovered
+      end
+
+      private
+
+      # Process one chunk at given offset
+      #
+      # For each chunk, we:
+      # 1. Compute b_vector chunks from recovery blocks
+      # 2. Subtract present block contributions to get final b
+      # 3. Apply A^-1 matrix to b to get recovered chunks
+      #
+      # @param recovered [Hash<Integer, String>] Recovered blocks being built
+      # @param offset [Integer] Current offset within blocks
+      # @param length [Integer] Chunk length
+      def process_chunk_at_offset(recovered, offset, length)
+        # Step 1: Initialize b vector from recovery blocks
+        # b_vector[i] starts as recovery[i] (for each recovery exponent used)
+        b_vector = compute_b_vector_chunks(offset, length)
+
+        # Step 2: Apply A^-1 to b vector to get recovered chunks
+        # For each missing block j: recovered[j] = sum(A^-1[j,i] * b[i])
+        apply_inverse_matrix(recovered, b_vector, offset, length)
+      end
+
+      # Compute b vector chunks
+      #
+      # b[i] = recovery[i] - sum(present[k] * base[present[k]]^exponent[i])
+      #
+      # @param offset [Integer] Offset within blocks
+      # @param length [Integer] Chunk length
+      # @return [Array<String>] B vector chunks (one per recovery block used)
+      def compute_b_vector_chunks(offset, length)
+        b_chunks = []
+
+        # For each recovery block being used (must match matrix computation!)
+        matrix.used_recovery_exponents.each_with_index do |exponent, _exp_idx|
+          # Start with recovery block chunk
+          recovery_data = recovery_blocks[exponent]
+          b_chunk = recovery_data[offset, length].dup
+
+          # Subtract contributions from present blocks
+          present_blocks.each do |present_idx, present_data|
+            coefficient = matrix.present_contribution_coefficient(present_idx,
+                                                                  exponent)
+            next if coefficient.zero?
+
+            present_chunk = present_data[offset, length]
+
+            # Subtract: b -= present * coefficient
+            # In GF(2^16), subtraction is XOR, so: b ^= present * coefficient
+            subtract_contribution(b_chunk, present_chunk, coefficient, length)
+          end
+
+          b_chunks << b_chunk
+        end
+
+        b_chunks
+      end
+
+      # Subtract present block contribution from b chunk
+      #
+      # b_chunk ^= present_chunk * coefficient (GF subtraction is XOR)
+      #
+      # @param b_chunk [String] B vector chunk (modified in place)
+      # @param present_chunk [String] Present block chunk
+      # @param coefficient [Integer] Galois field coefficient
+      # @param length [Integer] Chunk length
+      def subtract_contribution(b_chunk, present_chunk, coefficient, length)
+        num_words = length / 2
+
+        num_words.times do |i|
+          word_offset = i * 2
+
+          # Read present word
+          present_word = present_chunk.getbyte(word_offset) |
+            (present_chunk.getbyte(word_offset + 1) << 8)
+
+          # Read current b word
+          b_word = b_chunk.getbyte(word_offset) |
+            (b_chunk.getbyte(word_offset + 1) << 8)
+
+          # Compute: b ^= present * coefficient
+          contribution = Galois16.multiply(present_word, coefficient)
+          result = Galois16.add(b_word, contribution) # add is XOR in GF(2^16)
+
+          # Write back
+          b_chunk.setbyte(word_offset, result & 0xFF)
+          b_chunk.setbyte(word_offset + 1, (result >> 8) & 0xFF)
+        end
+      end
+
+      # Apply inverse matrix to b vector
+      #
+      # For each missing block j:
+      #   recovered[j] += sum_i(A^-1[j,i] * b[i])
+      #
+      # @param recovered [Hash<Integer, String>] Recovered blocks being built
+      # @param b_vector [Array<String>] B vector chunks
+      # @param offset [Integer] Offset within blocks
+      # @param length [Integer] Chunk length
+      def apply_inverse_matrix(recovered, b_vector, offset, length)
+        # For each output (missing block)
+        missing_indices.each_with_index do |missing_idx, output_idx|
+          output_block = recovered[missing_idx]
+
+          # For each b vector element (recovery block)
+          b_vector.each_with_index do |b_chunk, recovery_idx|
+            # Get coefficient from A^-1
+            coefficient = matrix.coefficient(output_idx, recovery_idx)
+            next if coefficient.zero?
+
+            # CRITICAL: Accumulate chunk at the correct offset
+            matrix.process_chunk(coefficient, b_chunk, output_block, length,
+                                 output_offset: offset)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/parity/galois16.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Parity
+    # Pure implementation of Galois Field GF(2^16) arithmetic
+    # Uses generator polynomial 0x1100B (69643) as per PAR2 specification
+    #
+    # This is pure algorithm code with no I/O dependencies.
+    # All operations are exact (no floating point) and work in GF(2^16).
+    class Galois16
+      BITS = 16
+      GENERATOR = 0x1100B # 69643 in decimal
+      FIELD_SIZE = 1 << BITS # 65536
+      LIMIT = FIELD_SIZE - 1 # 65535
+
+      # Lookup tables for efficient operations
+      @log_table = nil
+      @antilog_table = nil
+
+      class << self
+        # Build log and antilog tables
+        # This is called once when the class is loaded
+        def build_tables
+          return if @log_table && @antilog_table
+
+          @log_table = Array.new(FIELD_SIZE)
+          @antilog_table = Array.new(FIELD_SIZE)
+
+          b = 1
+          LIMIT.times do |l|
+            @log_table[b] = l
+            @antilog_table[l] = b
+
+            b <<= 1
+            b ^= GENERATOR if b.anybits?(FIELD_SIZE)
+          end
+
+          # Special cases for zero
+          @log_table[0] = LIMIT
+          @antilog_table[LIMIT] = 0
+        end
+
+        # Get log value (discrete logarithm)
+        def log(value)
+          @log_table[value & 0xFFFF]
+        end
+
+        # Get antilog value (inverse of log)
+        def antilog(value)
+          @antilog_table[value % FIELD_SIZE]
+        end
+
+        # Addition in GF(2^16) - same as subtraction (XOR)
+        def add(a, b)
+          (a ^ b) & 0xFFFF
+        end
+
+        # Subtraction in GF(2^16) - same as addition (XOR)
+        alias subtract add
+
+        # Multiplication in GF(2^16)
+        def multiply(a, b)
+          a &= 0xFFFF
+          b &= 0xFFFF
+
+          return 0 if a.zero? || b.zero?
+
+          sum = @log_table[a] + @log_table[b]
+          sum -= LIMIT if sum >= LIMIT
+          @antilog_table[sum]
+        end
+
+        # Division in GF(2^16)
+        def divide(a, b)
+          a &= 0xFFFF
+          b &= 0xFFFF
+
+          return 0 if a.zero?
+          raise ArgumentError, "Division by zero in GF(2^16)" if b.zero?
+
+          diff = @log_table[a] - @log_table[b]
+          diff += LIMIT if diff.negative?
+          @antilog_table[diff]
+        end
+
+        # Power in GF(2^16): compute a^n
+        def power(a, n)
+          a &= 0xFFFF
+
+          return 1 if n.zero?
+          return 0 if a.zero?
+
+          product = @log_table[a] * n
+
+          # Reduce modulo LIMIT using the identity:
+          # product mod LIMIT = (product >> BITS) + (product & LIMIT)
+          product = (product >> BITS) + (product & LIMIT)
+          product -= LIMIT if product >= LIMIT
+
+          @antilog_table[product]
+        end
+
+        # Compute GCD using Euclidean algorithm
+        def gcd(a, b)
+          while a != 0 && b != 0
+            if a > b
+              a %= b
+            else
+              b %= a
+            end
+          end
+          a + b
+        end
+
+        # Select base values for Reed-Solomon matrix
+        # Par2cmdline uses sequential logbases: base[i] = antilog[i]
+        # base[0] = antilog[0] = 1, base[1] = antilog[1] = 2, etc.
+        def select_bases(count)
+          raise ArgumentError, "Too many bases requested" if count >= LIMIT
+
+          bases = []
+
+          count.times do |i|
+            # Par2cmdline uses logbase = i (NOT i+1!)
+            # This gives: base[0]=1, base[1]=2, base[2]=4, etc.
+            logbase = i
+
+            if logbase >= LIMIT
+              raise ArgumentError,
+                    "Too many input blocks for Reed Solomon matrix"
+            end
+
+            # Convert log to actual value
+            bases << @antilog_table[logbase]
+          end
+
+          bases
+        end
+      end
+
+      # Build tables when class is loaded
+      build_tables
+    end
+  end
+end