omnizip 0.4.7-arm-linux

data/lib/omnizip/parallel/job_scheduler.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Parallel
+    # Job scheduler for load balancing and work distribution
+    #
+    # Manages job assignment to workers using different strategies:
+    # - Dynamic: Workers pull jobs as they become available (default)
+    # - Static: Pre-assign equal chunks to each worker
+    #
+    # @example Create scheduler with dynamic strategy
+    #   scheduler = Omnizip::Parallel::JobScheduler.new(strategy: :dynamic)
+    #   scheduler.schedule_jobs(jobs, worker_count: 4)
+    #
+    # @example Create scheduler with static strategy
+    #   scheduler = Omnizip::Parallel::JobScheduler.new(strategy: :static)
+    #   assignments = scheduler.schedule_jobs(jobs, worker_count: 4)
+    class JobScheduler
+      # @return [Symbol] scheduling strategy
+      attr_reader :strategy
+
+      # Initialize job scheduler
+      #
+      # @param strategy [Symbol] :dynamic or :static
+      def initialize(strategy: :dynamic)
+        @strategy = strategy
+        validate_strategy!
+      end
+
+      # Schedule jobs for workers
+      #
+      # @param jobs [Array] array of jobs to schedule
+      # @param worker_count [Integer] number of workers
+      # @return [Hash, Array] assignments (strategy-dependent)
+      def schedule_jobs(jobs, worker_count:)
+        case @strategy
+        when :dynamic
+          schedule_dynamic(jobs, worker_count)
+        when :static
+          schedule_static(jobs, worker_count)
+        when :round_robin
+          schedule_round_robin(jobs, worker_count)
+        when :bin_packing
+          schedule_bin_packing(jobs, worker_count)
+        end
+      end
+
+      # Estimate completion time based on job sizes and worker count
+      #
+      # @param jobs [Array] array of jobs with :size attribute
+      # @param worker_count [Integer] number of workers
+      # @param bytes_per_second [Float] processing rate
+      # @return [Float] estimated seconds to completion
+      def estimate_completion_time(jobs, worker_count:,
+bytes_per_second: 10_000_000)
+        total_bytes = jobs.sum { |job| job_size(job) }
+        return 0.0 if total_bytes.zero? || worker_count.zero?
+
+        # Simple estimate: total bytes / (workers * rate)
+        total_bytes.to_f / (worker_count * bytes_per_second)
+      end
+
+      # Calculate load balance quality metric
+      #
+      # @param assignments [Hash] worker_id => [jobs] mapping
+      # @return [Float] balance score (0.0 = perfect, 1.0 = worst)
+      def calculate_load_balance(assignments)
+        return 0.0 if assignments.empty?
+
+        # Calculate total size per worker
+        worker_sizes = assignments.transform_values do |jobs|
+          jobs.sum { |job| job_size(job) }
+        end
+
+        sizes = worker_sizes.values
+        return 0.0 if sizes.empty? || sizes.max.zero?
+
+        # Balance = (max - min) / max
+        (sizes.max - sizes.min).to_f / sizes.max
+      end
+
+      private
+
+      # Job size helper. Returns the job's #size if defined, else 1
+      # (counts as one unit of work). Centralized so the rest of the
+      # scheduler doesn't need +respond_to?+ checks.
+      #
+      # @param job [Object]
+      # @return [Integer]
+      def job_size(job)
+        if job.is_a?(::Numeric)
+          job
+        else
+          begin
+            job.size
+          rescue StandardError
+            nil
+          end || 1
+        end
+      end
+
+      # Validate scheduling strategy
+      #
+      # @raise [ArgumentError] if strategy is invalid
+      def validate_strategy!
+        valid_strategies = %i[dynamic static round_robin bin_packing]
+        return if valid_strategies.include?(@strategy)
+
+        raise ArgumentError,
+              "Invalid strategy: #{@strategy}. Must be one of: #{valid_strategies.join(', ')}"
+      end
+
+      # Dynamic scheduling: jobs pulled from queue as workers become available
+      #
+      # @param jobs [Array] jobs to schedule
+      # @param worker_count [Integer] number of workers
+      # @return [Hash] scheduling metadata
+      def schedule_dynamic(jobs, worker_count)
+        # In dynamic mode, we don't pre-assign jobs
+        # Workers pull from shared queue as they complete work
+        # Return metadata about the scheduling
+        {
+          strategy: :dynamic,
+          total_jobs: jobs.size,
+          worker_count: worker_count,
+          estimated_jobs_per_worker: (jobs.size.to_f / worker_count).ceil,
+          queue: jobs, # Jobs will be consumed from this queue
+        }
+      end
+
+      # Static scheduling: pre-assign jobs to workers in balanced chunks
+      #
+      # @param jobs [Array] jobs to schedule
+      # @param worker_count [Integer] number of workers
+      # @return [Hash] worker_id => [jobs] mapping
+      def schedule_static(jobs, worker_count)
+        return {} if jobs.empty? || worker_count.zero?
+
+        # Sort jobs by size (largest first) for better balance
+        sorted_jobs = jobs.sort_by do |job|
+          -job_size(job)
+        end
+
+        # Initialize worker assignments
+        assignments = (0...worker_count).to_h { |i| [i, []] }
+        worker_loads = Array.new(worker_count, 0)
+
+        # Assign each job to worker with smallest current load
+        sorted_jobs.each do |job|
+          job_size_value = job_size(job)
+
+          # Find worker with minimum load
+          min_worker = worker_loads.each_with_index.min_by { |load, _| load }[1]
+
+          # Assign job to this worker
+          assignments[min_worker] << job
+          worker_loads[min_worker] += job_size_value
+        end
+
+        # Add metadata
+        assignments[:metadata] = {
+          strategy: :static,
+          total_jobs: jobs.size,
+          worker_count: worker_count,
+          balance_score: calculate_load_balance(assignments.except(:metadata)),
+          worker_loads: worker_loads,
+        }
+
+        assignments
+      end
+
+      # Round-robin assignment (alternative simple strategy)
+      #
+      # @param jobs [Array] jobs to schedule
+      # @param worker_count [Integer] number of workers
+      # @return [Hash] worker_id => [jobs] mapping
+      def schedule_round_robin(jobs, worker_count)
+        return {} if jobs.empty? || worker_count.zero?
+
+        assignments = (0...worker_count).to_h { |i| [i, []] }
+
+        jobs.each_with_index do |job, index|
+          worker_id = index % worker_count
+          assignments[worker_id] << job
+        end
+
+        assignments
+      end
+
+      # Size-aware assignment with bin packing
+      #
+      # @param jobs [Array] jobs to schedule
+      # @param worker_count [Integer] number of workers
+      # @return [Hash] worker_id => [jobs] mapping
+      def schedule_bin_packing(jobs, worker_count)
+        return {} if jobs.empty? || worker_count.zero?
+
+        # Sort jobs by size (largest first)
+        sorted_jobs = jobs.sort_by do |job|
+          -job_size(job)
+        end
+
+        # First-fit decreasing bin packing
+        bins = Array.new(worker_count) { { jobs: [], total_size: 0 } }
+
+        sorted_jobs.each do |job|
+          job_size_value = job_size(job)
+
+          # Find bin with minimum total size
+          min_bin = bins.min_by { |bin| bin[:total_size] }
+          min_bin[:jobs] << job
+          min_bin[:total_size] += job_size_value
+        end
+
+        # Convert to standard format
+        assignments = {}
+        bins.each_with_index do |bin, index|
+          assignments[index] = bin[:jobs]
+        end
+
+        assignments
+      end
+    end
+  end
+end

data/lib/omnizip/parallel/parallel_compressor.rb

@@ -0,0 +1,326 @@
+# frozen_string_literal: true
+
+require "fractor"
+require "fileutils"
+
+module Omnizip
+  module Parallel
+    # Parallel compression coordinator using Fractor
+    #
+    # Manages parallel compression of files in a directory.
+    # Distributes compression work across multiple workers and
+    # writes results to archive in a thread-safe manner.
+    #
+    # @example Compress directory in parallel
+    #   compressor = Omnizip::Parallel::ParallelCompressor.new(threads: 4)
+    #   compressor.compress('files/', 'backup.zip')
+    #
+    # @example With options
+    #   options = Omnizip::Models::ParallelOptions.new
+    #   options.threads = 8
+    #   compressor = Omnizip::Parallel::ParallelCompressor.new(options)
+    #   compressor.compress('files/', 'backup.zip', compression: :lzma2)
+    class ParallelCompressor
+      # Fractor Work class for compression jobs
+      class CompressionWork < Fractor::Work
+        def initialize(file_path:, archive_path:, compression: :deflate,
+level: 6)
+          super({
+            file_path: file_path,
+            archive_path: archive_path,
+            compression: compression,
+            level: level,
+          })
+        end
+
+        def file_path
+          input[:file_path]
+        end
+
+        def archive_path
+          input[:archive_path]
+        end
+
+        def compression
+          input[:compression]
+        end
+
+        def level
+          input[:level]
+        end
+      end
+
+      # Fractor Worker class for compression
+      class CompressionWorker < Fractor::Worker
+        def process(work)
+          file_path = work.file_path
+          archive_path = work.archive_path
+          compression = work.compression
+          level = work.level
+
+          # Read file data
+          data = ::File.binread(file_path)
+          stat = ::File.stat(file_path)
+
+          # Compress the data
+          compressed_data = compress_data(data, compression, level)
+
+          # Calculate CRC32
+          crc32 = Omnizip::Checksums::Crc32.new.tap do |c|
+            c.update(data)
+          end.finalize
+
+          # Return result
+          Fractor::WorkResult.new(
+            result: {
+              archive_path: archive_path,
+              file_path: file_path,
+              compressed_data: compressed_data,
+              uncompressed_size: data.bytesize,
+              compressed_size: compressed_data.bytesize,
+              crc32: crc32,
+              stat: stat,
+              compression: compression,
+            },
+            work: work,
+          )
+        rescue StandardError => e
+          Fractor::WorkResult.new(
+            error: e,
+            work: work,
+          )
+        end
+
+        private
+
+        def compress_data(data, method, level)
+          case method
+          when :store
+            data
+          when :deflate
+            require "zlib"
+            Zlib::Deflate.new(level, -Zlib::MAX_WBITS).deflate(data, Zlib::FINISH)
+          when :bzip2
+            Omnizip::AlgorithmRegistry.get(:bzip2).compress(data, level: level)
+          when :lzma
+            Omnizip::AlgorithmRegistry.get(:lzma).compress(data, level: level)
+          when :lzma2
+            Omnizip::AlgorithmRegistry.get(:lzma2).compress(data, level: level)
+          when :zstandard
+            Omnizip::AlgorithmRegistry.get(:zstandard).compress(data,
+                                                                level: level)
+          else
+            raise Omnizip::UnsupportedFormatError,
+                  "Unsupported compression: #{method}"
+          end
+        end
+      end
+
+      # @return [Omnizip::Models::ParallelOptions] parallel options
+      attr_reader :options
+
+      # @return [Hash] compression statistics
+      attr_reader :stats
+
+      # Initialize parallel compressor
+      #
+      # @param options [Omnizip::Models::ParallelOptions, Hash] parallel options
+      # @param threads [Integer] number of threads (overrides options)
+      def initialize(options = nil, threads: nil)
+        @options = case options
+                   when Omnizip::Models::ParallelOptions
+                     options.dup
+                   when Hash
+                     Omnizip::Models::ParallelOptions.new.apply(options)
+                   else
+                     Omnizip::Models::ParallelOptions.new
+                   end
+
+        @options.threads = threads if threads
+        @options.validate_options!
+
+        @stats = {
+          files_processed: 0,
+          bytes_processed: 0,
+          bytes_compressed: 0,
+          start_time: nil,
+          end_time: nil,
+        }
+      end
+
+      # Compress directory to archive in parallel
+      #
+      # @param dir [String] directory path
+      # @param output [String] output archive path
+      # @param options [Hash] compression options
+      # @option options [Symbol] :compression compression method
+      # @option options [Integer] :level compression level
+      # @option options [Boolean] :recursive include subdirectories
+      # @option options [Proc] :progress progress callback
+      # @return [String] path to created archive
+      def compress(dir, output, **options)
+        unless ::File.exist?(dir)
+          raise Errno::ENOENT,
+                "Directory not found: #{dir}"
+        end
+        unless ::File.directory?(dir)
+          raise ArgumentError,
+                "Not a directory: #{dir}"
+        end
+
+        compression = options[:compression] || :deflate
+        level = options[:level] || 6
+        recursive = options.fetch(:recursive, true)
+        options[:progress]
+
+        @stats[:start_time] = Time.now
+
+        # Scan directory for files
+        files = scan_directory(dir, recursive: recursive)
+
+        # Create job queue
+        job_queue = JobQueue.new(max_size: @options.queue_size)
+
+        # Schedule jobs
+        JobScheduler.new(strategy: @options.strategy)
+        files.each do |file_path|
+          archive_path = file_path.sub("#{dir}/", "")
+          file_size = ::File.size(file_path)
+
+          job_queue.push_with_size(
+            file: file_path,
+            size: file_size,
+            data: {
+              archive_path: archive_path,
+              compression: compression,
+              level: level,
+            },
+          )
+        end
+
+        # Create work items from jobs
+        work_items = []
+        until job_queue.empty?
+          job = job_queue.pop(timeout: 0.1)
+          break unless job
+
+          work_items << CompressionWork.new(
+            file_path: job.file,
+            archive_path: job.data[:archive_path],
+            compression: job.data[:compression],
+            level: job.data[:level],
+          )
+        end
+
+        # Run the worker pool through the shared engine.
+        engine = Engine.new(worker_class: CompressionWorker,
+                            threads: @options.threads)
+        results = []
+        errors = engine.run(work_items) { |r| results << r }
+
+        # Handle errors
+        unless errors.empty?
+          error_msgs = errors.map do |e|
+            "#{e.work&.file_path}: #{e.error}"
+          end.join("\n")
+          raise Omnizip::CompressionError, "Compression errors:\n#{error_msgs}"
+        end
+
+        # Write archive sequentially (thread-safe)
+        write_archive(output, results, compression: compression)
+
+        @stats[:end_time] = Time.now
+        @stats[:files_processed] = results.size
+
+        output
+      end
+
+      # Get compression statistics
+      #
+      # @return [Hash] statistics
+      def statistics
+        duration = if @stats[:start_time] && @stats[:end_time]
+                     @stats[:end_time] - @stats[:start_time]
+                   else
+                     0
+                   end
+
+        @stats.merge(
+          duration: duration,
+          compression_ratio: calculate_compression_ratio,
+          throughput_mbps: calculate_throughput(duration),
+        )
+      end
+
+      private
+
+      # Scan directory for files
+      #
+      # @param dir [String] directory path
+      # @param recursive [Boolean] scan recursively
+      # @return [Array<String>] file paths
+      def scan_directory(dir, recursive: true)
+        files = []
+
+        if recursive
+          Dir.glob(::File.join(dir, "**", "*")).each do |path|
+            files << path if ::File.file?(path)
+          end
+        else
+          Dir.glob(::File.join(dir, "*")).each do |path|
+            files << path if ::File.file?(path)
+          end
+        end
+
+        files.sort
+      end
+
+      # Write archive from compressed results
+      #
+      # @param output [String] output path
+      # @param results [Array] compression results
+      # @param compression [Symbol] compression method
+      def write_archive(output, results, compression:)
+        writer = Omnizip::Formats::Zip::Writer.new(output)
+
+        results.each do |work_result|
+          result = work_result.result
+          next unless result
+
+          writer.add_precompressed_entry(
+            filename: result[:archive_path],
+            uncompressed_size: result[:uncompressed_size],
+            compressed_size: result[:compressed_size],
+            crc32: result[:crc32],
+            compressed_data: result[:compressed_data],
+            stat: result[:stat],
+          )
+
+          @stats[:bytes_processed] += result[:uncompressed_size]
+          @stats[:bytes_compressed] += result[:compressed_size]
+        end
+
+        writer.write_precompressed(compression_method: compression)
+      end
+
+      # Calculate compression ratio
+      #
+      # @return [Float] compression ratio percentage
+      def calculate_compression_ratio
+        return 0.0 if @stats[:bytes_processed].zero?
+
+        (1.0 - (@stats[:bytes_compressed].to_f / @stats[:bytes_processed])) * 100.0
+      end
+
+      # Calculate throughput in MB/s
+      #
+      # @param duration [Float] duration in seconds
+      # @return [Float] throughput in MB/s
+      def calculate_throughput(duration)
+        return 0.0 if duration.zero?
+
+        (@stats[:bytes_processed].to_f / (1024 * 1024)) / duration
+      end
+    end
+  end
+end