parallel 1.6.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 0f363b126a1d42e09e74864a71c940edaf139d79
-  data.tar.gz: a8976de6367828f3ef06b8eb81822515817b1afc
+SHA256:
+  metadata.gz: 3331bc6634cd376e4f3f8511049e49a0d4501110aaeac395e2f0c0b638f4a402
+  data.tar.gz: 357e1424cb8297b6472c2c5b9486e26da3969edd928be7d6ca7f21670211a8e0
 SHA512:
-  metadata.gz: 0a78ebb2abf7b12749b4ffb318eb70b8b04f00af516dcf438423080a26136bd2a87a097e9b593130294eaf4e9a1e97a754da7c8f7a7a46c7d7b84f5446bdd224
-  data.tar.gz: 3eb8871382f2669e89fecca80bc2dfe198c05df133795e55fd12bd283d0760d7a9c5035c2abc337431b57048647b343ea9022a3ca83bf4da875c53d8a47c3b0c
+  metadata.gz: 1885d4f814023905f76105f0edce9155024478408be851a4f6869e537400e924e03d34196e03b428803c274b5fbf7e9bf2d93f4548782aa6d3015063c7ed7883
+  data.tar.gz: 97d22f6b0320a089a3584e76c0baa245811861776e7337e913961cd64cfe0770131493d7de7e1355a62d6339d47d0b1f3ad32dd89084f31c69a61fcfab9f3148

data/lib/parallel/serializer.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+require 'openssl'
+require 'securerandom'
+
+module Parallel
+  # Pluggable wire serializers. Each must respond to `dump(data, io)` /
+  # `load(io)` (used directly by Worker) and `dump(data)` / `load(string)`
+  # (used by wrappers like Hmac).
+  module Serializer
+    # Raw Marshal. Fast but trusts anything written to the pipe — a same-UID
+    # attacker that reopens /proc/<pid>/fd/<n> can inject Marshal gadgets (RCE).
+    Marshal = ::Marshal
+
+    # Wraps any inner serializer with a length-prefixed HMAC-SHA256 frame keyed
+    # on a per-worker secret generated before fork. Forged frames from a
+    # pipe-injector fail verification.
+    class Hmac
+      LENGTH_FORMAT = 'N' # 32-bit big-endian unsigned int
+      LENGTH_BYTES = 4
+      MAC_BYTES = 32 # SHA256
+
+      def initialize(inner: Marshal, secret: SecureRandom.bytes(32))
+        @inner = inner
+        @secret = secret
+      end
+
+      def dump(data, io)
+        payload = @inner.dump(data)
+        mac = OpenSSL::HMAC.digest('SHA256', @secret, payload)
+        io.write([payload.bytesize].pack(LENGTH_FORMAT), mac, payload)
+      end
+
+      def load(io)
+        # nil at frame boundary = clean EOF (worker died / pipe closed between messages)
+        header = io.read(LENGTH_BYTES) || raise(EOFError) # eof stops worker
+        raise SecurityError, "truncated frame header" if header.bytesize != LENGTH_BYTES
+
+        length = header.unpack1(LENGTH_FORMAT)
+        mac = io.read(MAC_BYTES)
+        raise SecurityError, "truncated frame mac" if mac.nil? || mac.bytesize != MAC_BYTES
+
+        payload = io.read(length)
+        raise SecurityError, "truncated frame payload" if payload.nil? || payload.bytesize != length
+
+        expected = OpenSSL::HMAC.digest('SHA256', @secret, payload)
+        raise SecurityError, "HMAC mismatch on worker pipe" unless OpenSSL.fixed_length_secure_compare(mac, expected)
+
+        @inner.load(payload)
+      end
+    end
+  end
+end

data/lib/parallel/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Parallel
-  VERSION = Version = '1.6.0'
+  VERSION = Version = '2.1.0' # rubocop:disable Naming/ConstantName
 end

data/lib/parallel.rb

@@ -1,66 +1,111 @@
+# frozen_string_literal: true
 require 'rbconfig'
 require 'parallel/version'
-require 'parallel/processor_count'
+require 'parallel/serializer'
 
 module Parallel
-  extend Parallel::ProcessorCount
+  Stop = Object.new.freeze
 
   class DeadWorker < StandardError
   end
 
   class Break < StandardError
+    attr_reader :value
+
+    def initialize(value = nil)
+      super()
+      @value = value
+    end
+
+    # marshal_dump that is used for ruby exceptions
+    # avoid dumping the cause since nobody needs that and it can include undumpable exceptions
+    def _dump(_depth)
+      Marshal.dump(@value)
+    end
+
+    # marshal_load that is used for ruby exceptions
+    def self._load(data)
+      new(Marshal.load(data))
+    end
   end
 
-  class Kill < StandardError
+  class Kill < Break
   end
 
-  Stop = Object.new
+  class UndumpableException < StandardError
+    attr_reader :backtrace
+
+    def initialize(original)
+      super("#{original.class}: #{original.message}")
+      @backtrace = original.backtrace
+    end
+  end
 
   class ExceptionWrapper
     attr_reader :exception
+
     def initialize(exception)
-      dumpable = Marshal.dump(exception) rescue nil
-      unless dumpable
-        exception = RuntimeError.new("Undumpable Exception -- #{exception.inspect}")
+      # Remove the bindings stack added by the better_errors gem,
+      # because it cannot be marshalled
+      if exception.instance_variable_defined? :@__better_errors_bindings_stack
+        exception.send :remove_instance_variable, :@__better_errors_bindings_stack
       end
 
-      @exception = exception
+      @exception =
+        begin
+          Marshal.dump(exception) && exception
+        rescue StandardError
+          UndumpableException.new(exception)
+        end
     end
   end
 
   class Worker
     attr_reader :pid, :read, :write
     attr_accessor :thread
-    def initialize(read, write, pid)
-      @read, @write, @pid = read, write, pid
+
+    def initialize(read, write, pid, serializer)
+      @read = read
+      @write = write
+      @pid = pid
+      @serializer = serializer
     end
 
-    def close_pipes
-      read.close
-      write.close
+    def stop
+      close_pipes
+      wait # if it goes zombie, rather wait here to be able to debug
     end
 
-    def wait
-      Process.wait(pid)
-    rescue Interrupt
-      # process died
+    # might be passed to started_processes and simultaneously closed by another thread
+    # when running in isolation mode, so we have to check if it is closed before closing
+    def close_pipes
+      read.close unless read.closed?
+      write.close unless write.closed?
     end
 
     def work(data)
       begin
-        Marshal.dump(data, write)
+        @serializer.dump(data, write)
       rescue Errno::EPIPE
         raise DeadWorker
       end
 
       result = begin
-        Marshal.load(read)
+        @serializer.load(read)
       rescue EOFError
         raise DeadWorker
       end
-      raise result.exception if ExceptionWrapper === result
+      raise result.exception if result.is_a?(ExceptionWrapper)
       result
     end
+
+    private
+
+    def wait
+      Process.wait(pid)
+    rescue Interrupt
+      # process died
+    end
   end
 
   class JobFactory
@@ -79,7 +124,7 @@ module Parallel
         item, index = @mutex.synchronize do
           return if @stopped
           item = @lambda.call
-          @stopped = (item == Parallel::Stop)
+          @stopped = (item == Stop)
           return if @stopped
           [item, @index += 1]
         end
@@ -99,10 +144,13 @@ module Parallel
       end
     end
 
+    # generate item that is sent to workers
+    # just index is faster + less likely to blow up with unserializable errors
     def pack(item, index)
       producer? ? [item, index] : index
     end
 
+    # unpack item that is sent to workers
     def unpack(data)
       producer? ? data : [@source[data], data]
     end
@@ -114,7 +162,7 @@ module Parallel
     end
 
     def queue_wrapper(array)
-      array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) }
+      array.respond_to?(:num_waiting) && array.respond_to?(:pop) && -> { array.pop(false) }
     end
   end
 
@@ -130,7 +178,7 @@ module Parallel
 
         if @to_be_killed.empty?
           old_interrupt = trap_interrupt(signal) do
-            $stderr.puts 'Parallel execution interrupted, exiting ...'
+            warn 'Parallel execution interrupted, exiting ...'
             @to_be_killed.flatten.each { |pid| kill(pid) }
           end
         end
@@ -157,7 +205,7 @@ module Parallel
 
         Signal.trap signal do
           yield
-          if old == "DEFAULT"
+          if !old || old == "DEFAULT"
             raise Interrupt
           else
             old.call
@@ -174,37 +222,61 @@ module Parallel
   end
 
   class << self
-    def in_threads(options={:count => 2})
-      count, _ = extract_count_from_options(options)
-      Array.new(count).each_with_index.map do |_, i|
-        Thread.new { yield(i) }
-      end.map!(&:value)
+    def in_threads(options = { count: 2 })
+      threads = []
+      count, = extract_count_from_options(options)
+
+      Thread.handle_interrupt(Exception => :never) do
+        Thread.handle_interrupt(Exception => :immediate) do
+          count.times do |i|
+            threads << Thread.new { yield(i) }
+          end
+          threads.map(&:value)
+        end
+      ensure
+        threads.each(&:kill)
+      end
     end
 
     def in_processes(options = {}, &block)
       count, options = extract_count_from_options(options)
       count ||= processor_count
-      map(0...count, options.merge(:in_processes => count), &block)
+      map(0...count, options.merge(in_processes: count), &block)
     end
 
-    def each(array, options={}, &block)
-      map(array, options.merge(:preserve_results => false), &block)
-      array
+    def each(array, options = {}, &block)
+      map(array, options.merge(preserve_results: false), &block)
     end
 
-    def each_with_index(array, options={}, &block)
-      each(array, options.merge(:with_index => true), &block)
+    def any?(*args, &block)
+      raise "You must provide a block when calling #any?" if block.nil?
+      !each(*args) { |*a| raise Kill if block.call(*a) }
+    end
+
+    def all?(*args, &block)
+      raise "You must provide a block when calling #all?" if block.nil?
+      !!each(*args) { |*a| raise Kill unless block.call(*a) }
+    end
+
+    def each_with_index(array, options = {}, &block)
+      each(array, options.merge(with_index: true), &block)
     end
 
     def map(source, options = {}, &block)
+      options = options.dup
       options[:mutex] = Mutex.new
 
-      if RUBY_PLATFORM =~ /java/ and not options[:in_processes]
+      if options[:in_processes] && options[:in_threads]
+        raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`."
+      elsif RUBY_PLATFORM.include?('java') && !options[:in_processes]
         method = :in_threads
         size = options[method] || processor_count
       elsif options[:in_threads]
         method = :in_threads
         size = options[method]
+      elsif options[:in_ractors]
+        method = :in_ractors
+        size = options[method]
       else
         method = :in_processes
         if Process.respond_to?(:fork)
@@ -221,27 +293,115 @@ module Parallel
       options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
       add_progress_bar!(job_factory, options)
 
-      if size == 0
-        work_direct(job_factory, options, &block)
-      elsif method == :in_threads
-        work_in_threads(job_factory, options.merge(:count => size), &block)
-      else
-        work_in_processes(job_factory, options.merge(:count => size), &block)
+      result =
+        if size == 0
+          work_direct(job_factory, options, &block)
+        elsif method == :in_threads
+          work_in_threads(job_factory, options.merge(count: size), &block)
+        elsif method == :in_ractors
+          work_in_ractors(job_factory, options.merge(count: size), &block)
+        else
+          work_in_processes(job_factory, options.merge(count: size), &block)
+        end
+
+      return result.value if result.is_a?(Break)
+      raise result if result.is_a?(Exception)
+      options[:return_results] ? result : source
+    end
+
+    def map_with_index(array, options = {}, &block)
+      map(array, options.merge(with_index: true), &block)
+    end
+
+    def flat_map(...)
+      map(...).flatten(1)
+    end
+
+    def filter_map(...)
+      map(...).compact
+    end
+
+    # Number of physical processor cores on the current system.
+    def physical_processor_count
+      @physical_processor_count ||= begin
+        ppc =
+          case RbConfig::CONFIG["target_os"]
+          when /darwin[12]/
+            IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
+          when /linux/
+            cores = {} # unique physical ID / core ID combinations
+            phy = 0
+            File.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
+              if ln.start_with?("physical")
+                phy = ln[/\d+/]
+              elsif ln.start_with?("core")
+                cid = "#{phy}:#{ln[/\d+/]}"
+                cores[cid] = true unless cores[cid]
+              end
+            end
+            cores.count
+          when /mswin|mingw/
+            physical_processor_count_windows
+          else
+            processor_count
+          end
+        # fall back to logical count if physical info is invalid
+        ppc > 0 ? ppc : processor_count
       end
     end
 
-    def map_with_index(array, options={}, &block)
-      map(array, options.merge(:with_index => true), &block)
+    # Number of processors seen by the OS or value considering CPU quota if the process is inside a cgroup,
+    # used for process scheduling
+    def processor_count
+      @processor_count ||= Integer(ENV['PARALLEL_PROCESSOR_COUNT'] || available_processor_count)
+    end
+
+    def worker_number
+      Thread.current[:parallel_worker_number]
+    end
+
+    # TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
+    def worker_number=(worker_num)
+      Thread.current[:parallel_worker_number] = worker_num
     end
 
     private
 
+    def physical_processor_count_windows
+      # Get-CimInstance introduced in PowerShell 3 or earlier: https://learn.microsoft.com/en-us/previous-versions/powershell/module/cimcmdlets/get-ciminstance?view=powershell-3.0
+      result = run(
+        'powershell -command "Get-CimInstance -ClassName Win32_Processor -Property NumberOfCores ' \
+        '| Select-Object -Property NumberOfCores"'
+      )
+      if !result || $?.exitstatus != 0
+        # fallback to deprecated wmic for older systems
+        result = run("wmic cpu get NumberOfCores")
+      end
+      if !result || $?.exitstatus != 0
+        # Bail out if both commands returned something unexpected
+        warn "guessing pyhsical processor count"
+        processor_count
+      else
+        # powershell: "\nNumberOfCores\n-------------\n            4\n\n\n"
+        # wmic:       "NumberOfCores  \n\n4              \n\n\n\n"
+        result.scan(/\d+/).map(&:to_i).reduce(:+)
+      end
+    end
+
+    def run(command)
+      IO.popen(command, &:read)
+    rescue Errno::ENOENT
+      # Ignore
+    end
+
     def add_progress_bar!(job_factory, options)
-      if progress_options = options[:progress]
+      if (progress_options = options[:progress])
         raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
         require 'ruby-progressbar'
 
-        if progress_options.respond_to? :to_str
+        if progress_options == true
+          progress_options = { title: "Progress" }
+        elsif progress_options.respond_to? :to_str
           progress_options = { title: progress_options.to_str }
         end
 
@@ -260,47 +420,150 @@ module Parallel
     end
 
     def work_direct(job_factory, options, &block)
+      self.worker_number = 0
       results = []
-      while set = job_factory.next
-        item, index = set
-        results << with_instrumentation(item, index, options) do
-          call_with_index(item, index, options, &block)
+      exception = nil
+      begin
+        while (set = job_factory.next)
+          item, index = set
+          results << with_instrumentation(item, index, options) do
+            call_with_index(item, index, options, &block)
+          end
         end
+      rescue StandardError
+        exception = $!
       end
-      results
+      exception || results
+    ensure
+      self.worker_number = nil
     end
 
     def work_in_threads(job_factory, options, &block)
       raise "interrupt_signal is no longer supported for threads" if options[:interrupt_signal]
       results = []
+      results_mutex = Mutex.new # arrays are not thread-safe on jRuby
       exception = nil
 
-      in_threads(options) do
+      in_threads(options) do |worker_num|
+        self.worker_number = worker_num
         # as long as there are more jobs, work on one of them
-        while !exception && set = job_factory.next
+        while !exception && (set = job_factory.next)
           begin
             item, index = set
-            results[index] = with_instrumentation item, index, options do
+            result = with_instrumentation item, index, options do
               call_with_index(item, index, options, &block)
             end
+            results_mutex.synchronize { results[index] = result }
+          rescue StandardError
+            exception = $!
+          end
+        end
+      end
+
+      exception || results
+    end
+
+    def work_in_ractors(job_factory, options)
+      exception = nil
+      results = []
+      results_mutex = Mutex.new # arrays are not thread-safe on jRuby
+
+      callback = options[:ractor]
+      if block_given? || !callback
+        raise ArgumentError, "pass the code you want to execute as `ractor: [ClassName, :method_name]`"
+      end
+
+      use_port = defined?(Ractor::Port)
+
+      # build
+      ports = {} # port (ruby 4+) or ractor (ruby 3) => ractor
+      options.fetch(:count).times do
+        port, ractor = ractor_build(use_port)
+        ports[port] = ractor
+      end
+
+      # start
+      ports.dup.each do |port, ractor|
+        if (job = job_factory.next)
+          item, index = job
+          instrument_start item, index, options
+          ractor.send [callback, item, index]
+        else # not enough work, `receive` would hang
+          ractor_stop ractor
+          ports.delete port
+        end
+      end
+
+      # receive result and send new items to done ractors
+      while (job = job_factory.next)
+        # receive result
+        done_port, (exception, result, item_prev, index_prev) = Ractor.select(*ports.keys)
+        done_ractor = ports[done_port]
+        if exception
+          ports.delete done_port
+          break
+        end
+        ractor_result item_prev, index_prev, result, results, results_mutex, options
+
+        # send new
+        item_next, index_next = job
+        instrument_start item_next, index_next, options
+        done_ractor.send([callback, item_next, index_next])
+      end
+
+      # finish
+      ports.each do |port, ractor|
+        (new_exception, result, item, index) = use_port ? port.receive : ractor.take
+        exception ||= new_exception
+        next if new_exception
+        ractor_result item, index, result, results, results_mutex, options
+        ractor_stop ractor
+      end
+
+      exception || results
+    end
+
+    def ractor_build(use_port)
+      args = use_port ? [Ractor::Port.new] : []
+      ractor = Ractor.new(*args) do |port|
+        loop do
+          (klass, method_name), item, index = receive
+          break if index == :break
+          begin
+            result = [nil, klass.send(method_name, item), item, index]
           rescue StandardError => e
-            exception = e
+            result = [e, nil, item, index]
+          end
+          if port
+            port.send result
+          else
+            Ractor.yield result
           end
         end
       end
+      [use_port ? args.first : ractor, ractor]
+    end
+
+    def ractor_result(item, index, result, results, results_mutex, options)
+      instrument_finish item, index, result, options
+      results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
+    end
 
-      handle_exception(exception, results)
+    def ractor_stop(ractor)
+      ractor.send([[nil, nil], nil, :break])
     end
 
     def work_in_processes(job_factory, options, &blk)
       workers = create_workers(job_factory, options, &blk)
       results = []
+      results_mutex = Mutex.new # arrays are not thread-safe
       exception = nil
 
       UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
         in_threads(options) do |i|
           worker = workers[i]
           worker.thread = Thread.current
+          worked = false
 
           begin
             loop do
@@ -308,34 +571,52 @@ module Parallel
               item, index = job_factory.next
               break unless index
 
+              if options[:isolation]
+                worker = replace_worker(job_factory, workers, i, options, blk) if worked
+                worked = true
+                worker.thread = Thread.current
+              end
+
               begin
-                results[index] = with_instrumentation item, index, options do
+                result = with_instrumentation item, index, options do
                   worker.work(job_factory.pack(item, index))
                 end
+                results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
               rescue StandardError => e
                 exception = e
-                if Parallel::Kill === exception
+                if exception.is_a?(Kill)
                   (workers - [worker]).each do |w|
-                    w.thread.kill
+                    w.thread&.kill
                     UserInterruptHandler.kill(w.pid)
                   end
                 end
               end
             end
           ensure
-            worker.close_pipes
-            worker.wait # if it goes zombie, rather wait here to be able to debug
+            worker.stop
           end
         end
       end
 
-      handle_exception(exception, results)
+      exception || results
+    end
+
+    def replace_worker(job_factory, workers, index, options, blk)
+      options[:mutex].synchronize do
+        # old worker is no longer used ... stop it
+        worker = workers[index]
+        worker.stop
+
+        # create a new replacement worker
+        running = workers - [worker]
+        workers[index] = worker(job_factory, options.merge(started_workers: running, worker_number: index), &blk)
+      end
     end
 
     def create_workers(job_factory, options, &block)
       workers = []
-      Array.new(options[:count]).each do
-        workers << worker(job_factory, options.merge(:started_workers => workers), &block)
+      Array.new(options[:count]).each_with_index do |_, i|
+        workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block)
       end
       workers
     end
@@ -343,8 +624,11 @@ module Parallel
     def worker(job_factory, options, &block)
       child_read, parent_write = IO.pipe
       parent_read, child_write = IO.pipe
+      options[:serializer] ||= Serializer::Marshal
 
       pid = Process.fork do
+        self.worker_number = options[:worker_number]
+
         begin
           options.delete(:started_workers).each(&:close_pipes)
 
@@ -361,28 +645,33 @@ module Parallel
       child_read.close
       child_write.close
 
-      Worker.new(parent_read, parent_write, pid)
+      Worker.new(parent_read, parent_write, pid, options[:serializer])
     end
 
     def process_incoming_jobs(read, write, job_factory, options, &block)
+      serializer = options.fetch(:serializer)
       until read.eof?
-        data = Marshal.load(read)
+        data = serializer.load(read)
         item, index = job_factory.unpack(data)
-        result = begin
-          call_with_index(item, index, options, &block)
-        rescue StandardError => e
-          ExceptionWrapper.new(e)
+
+        result =
+          begin
+            call_with_index(item, index, options, &block)
+          # https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
+          rescue NoMemoryError, SignalException, Interrupt, SystemExit # rubocop:disable Lint/ShadowedException
+            raise $!
+          rescue Exception # rubocop:disable Lint/RescueException
+            ExceptionWrapper.new($!)
+          end
+
+        begin
+          serializer.dump(result, write)
+        rescue Errno::EPIPE
+          return # parent thread already dead
         end
-        Marshal.dump(result, write)
       end
     end
 
-    def handle_exception(exception, results)
-      return nil if [Parallel::Break, Parallel::Kill].include? exception.class
-      raise exception if exception
-      results
-    end
-
     # options is either a Integer or a Hash with :count
     def extract_count_from_options(options)
       if options.is_a?(Hash)
@@ -397,22 +686,62 @@ module Parallel
     def call_with_index(item, index, options, &block)
       args = [item]
       args << index if options[:with_index]
+      results = block.call(*args)
       if options[:return_results]
-        block.call(*args)
+        results
       else
-        block.call(*args)
         nil # avoid GC overhead of passing large results around
       end
     end
 
     def with_instrumentation(item, index, options)
-      on_start = options[:start]
-      on_finish = options[:finish]
-      options[:mutex].synchronize { on_start.call(item, index) } if on_start
+      instrument_start(item, index, options)
       result = yield
+      instrument_finish(item, index, result, options)
       result unless options[:preserve_results] == false
-    ensure
-      options[:mutex].synchronize { on_finish.call(item, index, result) } if on_finish
+    end
+
+    def instrument_finish(item, index, result, options)
+      return unless (on_finish = options[:finish])
+      return instrument_finish_in_order(item, index, result, options) if options[:finish_in_order]
+      options[:mutex].synchronize { on_finish.call(item, index, result) }
+    end
+
+    # yield results in the order of the input items
+    # needs to use `options` to store state between executions
+    # needs to use `done` index since a nil result would also be valid
+    def instrument_finish_in_order(item, index, result, options)
+      options[:mutex].synchronize do
+        # initialize our state
+        options[:finish_done] ||= []
+        options[:finish_expecting] ||= 0 # we wait for item at index 0
+
+        # store current result
+        options[:finish_done][index] = [item, result]
+
+        # yield all results that are now in order
+        break unless index == options[:finish_expecting]
+        index.upto(options[:finish_done].size).each do |i|
+          break unless (done = options[:finish_done][i])
+          options[:finish_done][i] = nil # allow GC to free this item and result
+          options[:finish].call(done[0], i, done[1])
+          options[:finish_expecting] += 1
+        end
+      end
+    end
+
+    def instrument_start(item, index, options)
+      return unless (on_start = options[:start])
+      options[:mutex].synchronize { on_start.call(item, index) }
+    end
+
+    def available_processor_count
+      gem 'concurrent-ruby', '>= 1.3.4'
+      require 'concurrent-ruby'
+      Concurrent.available_processor_count.floor
+    rescue LoadError
+      require 'etc'
+      Etc.nprocessors
     end
   end
 end

metadata

@@ -1,16 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parallel
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Michael Grosser
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-26 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
-description: 
 email: michael@grosser.it
 executables: []
 extensions: []
@@ -18,13 +16,18 @@ extra_rdoc_files: []
 files:
 - MIT-LICENSE.txt
 - lib/parallel.rb
-- lib/parallel/processor_count.rb
+- lib/parallel/serializer.rb
 - lib/parallel/version.rb
 homepage: https://github.com/grosser/parallel
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  bug_tracker_uri: https://github.com/grosser/parallel/issues
+  documentation_uri: https://github.com/grosser/parallel/blob/v2.1.0/Readme.md
+  source_code_uri: https://github.com/grosser/parallel/tree/v2.1.0
+  wiki_uri: https://github.com/grosser/parallel/wiki
+  changelog_uri: https://github.com/grosser/parallel/blob/v2.1.0/CHANGELOG.md
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -32,16 +35,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Run any kind of code in parallel processes
 test_files: []

data/lib/parallel/processor_count.rb

@@ -1,85 +0,0 @@
-module Parallel
-  module ProcessorCount
-    # Number of processors seen by the OS and used for process scheduling.
-    #
-    # * AIX: /usr/sbin/pmcycles (AIX 5+), /usr/sbin/lsdev
-    # * BSD: /sbin/sysctl
-    # * Cygwin: /proc/cpuinfo
-    # * Darwin: /usr/bin/hwprefs, /usr/sbin/sysctl
-    # * HP-UX: /usr/sbin/ioscan
-    # * IRIX: /usr/sbin/sysconf
-    # * Linux: /proc/cpuinfo
-    # * Minix 3+: /proc/cpuinfo
-    # * Solaris: /usr/sbin/psrinfo
-    # * Tru64 UNIX: /usr/sbin/psrinfo
-    # * UnixWare: /usr/sbin/psrinfo
-    #
-    def processor_count
-      @processor_count ||= begin
-        os_name = RbConfig::CONFIG["target_os"]
-        if os_name =~ /mingw|mswin/
-          require 'win32ole'
-          result = WIN32OLE.connect("winmgmts://").ExecQuery(
-            "select NumberOfLogicalProcessors from Win32_Processor")
-          result.to_enum.collect(&:NumberOfLogicalProcessors).reduce(:+)
-        elsif File.readable?("/proc/cpuinfo")
-          IO.read("/proc/cpuinfo").scan(/^processor/).size
-        elsif File.executable?("/usr/bin/hwprefs")
-          IO.popen("/usr/bin/hwprefs thread_count").read.to_i
-        elsif File.executable?("/usr/sbin/psrinfo")
-          IO.popen("/usr/sbin/psrinfo").read.scan(/^.*on-*line/).size
-        elsif File.executable?("/usr/sbin/ioscan")
-          IO.popen("/usr/sbin/ioscan -kC processor") do |out|
-            out.read.scan(/^.*processor/).size
-          end
-        elsif File.executable?("/usr/sbin/pmcycles")
-          IO.popen("/usr/sbin/pmcycles -m").read.count("\n")
-        elsif File.executable?("/usr/sbin/lsdev")
-          IO.popen("/usr/sbin/lsdev -Cc processor -S 1").read.count("\n")
-        elsif File.executable?("/usr/sbin/sysconf") and os_name =~ /irix/i
-          IO.popen("/usr/sbin/sysconf NPROC_ONLN").read.to_i
-        elsif File.executable?("/usr/sbin/sysctl")
-          IO.popen("/usr/sbin/sysctl -n hw.ncpu").read.to_i
-        elsif File.executable?("/sbin/sysctl")
-          IO.popen("/sbin/sysctl -n hw.ncpu").read.to_i
-        else
-          $stderr.puts "Unknown platform: " + RbConfig::CONFIG["target_os"]
-          $stderr.puts "Assuming 1 processor."
-          1
-        end
-      end
-    end
-
-    # Number of physical processor cores on the current system.
-    #
-    def physical_processor_count
-      @physical_processor_count ||= begin
-        ppc = case RbConfig::CONFIG["target_os"]
-        when /darwin1/
-          IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
-        when /linux/
-          cores = {}  # unique physical ID / core ID combinations
-          phy = 0
-          IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
-            if ln.start_with?("physical")
-              phy = ln[/\d+/]
-            elsif ln.start_with?("core")
-              cid = phy + ":" + ln[/\d+/]
-              cores[cid] = true if not cores[cid]
-            end
-          end
-          cores.count
-        when /mswin|mingw/
-          require 'win32ole'
-          result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
-            "select NumberOfCores from Win32_Processor")
-          result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
-        else
-          processor_count
-        end
-        # fall back to logical count if physical info is invalid
-        ppc > 0 ? ppc : processor_count
-      end
-    end
-  end
-end