parallel 1.10.0 → 1.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: bbf5d8fc78e3bdf6f1d12ae80ffff41fa5e6d0ed
-  data.tar.gz: c1eb1b447d1c59dce453710d943d2b952d1079b3
+SHA256:
+  metadata.gz: f7b4a5662ef68724c75c2d655d9bd09ad109edc02d3199a0744093544608f74b
+  data.tar.gz: 247f7f3745c42cdafeb5d403fff250e99b99a56d4dc4948050dcf20ecb60d183
 SHA512:
-  metadata.gz: 0fbe56fc6a4ba0ec2df7bbdb9f16e44b08e42fc0bced3ff939a7baa128940d0afc6ae27da216ffb78dc066f55989e779198cb39137366e6988ab57ba854681cf
-  data.tar.gz: 1b5252fb4ec4ef143072dbd604641a5d614e16142065f1a1618f665cbc2f74074305adef81e784587ec6a582662bfde67654e0e431850cf9e31dbf0b8082b922
+  metadata.gz: 18f62ea9fefb30bda9b89f0189074da99572c5c17fb93ba28e553b0eb9f6d7a61bd70ba423004b138b05c3c86f8e04fd32fce81da40a68c0bdca39e263b8b892
+  data.tar.gz: dca61c9e3cab22de0ad064c96c55afc88177214108d6bdbfb8e9a93343f23d6ee321c83aa6c4cfa9ee33da79d01802d7b952f3dd0360f9cca26e5da78a4117fa

data/lib/parallel/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Parallel
-  VERSION = Version = '1.10.0'
+  VERSION = Version = '1.24.0' # rubocop:disable Naming/ConstantName
 end

data/lib/parallel.rb

@@ -1,39 +1,48 @@
+# frozen_string_literal: true
 require 'rbconfig'
 require 'parallel/version'
-require 'parallel/processor_count'
 
 module Parallel
-  extend Parallel::ProcessorCount
+  Stop = Object.new.freeze
 
   class DeadWorker < StandardError
   end
 
   class Break < StandardError
+    attr_reader :value
+
+    def initialize(value = nil)
+      super()
+      @value = value
+    end
   end
 
-  class Kill < StandardError
+  class Kill < Break
   end
 
   class UndumpableException < StandardError
+    attr_reader :backtrace
+
     def initialize(original)
       super "#{original.class}: #{original.message}"
-      @bracktrace = original.backtrace
-    end
-
-    def backtrace
-      @bracktrace
+      @backtrace = original.backtrace
     end
   end
 
-  Stop = Object.new
-
   class ExceptionWrapper
     attr_reader :exception
+
     def initialize(exception)
+      # Remove the bindings stack added by the better_errors gem,
+      # because it cannot be marshalled
+      if exception.instance_variable_defined? :@__better_errors_bindings_stack
+        exception.send :remove_instance_variable, :@__better_errors_bindings_stack
+      end
+
       @exception =
         begin
           Marshal.dump(exception) && exception
-        rescue
+        rescue StandardError
           UndumpableException.new(exception)
         end
     end
@@ -42,8 +51,11 @@ module Parallel
   class Worker
     attr_reader :pid, :read, :write
     attr_accessor :thread
+
     def initialize(read, write, pid)
-      @read, @write, @pid = read, write, pid
+      @read = read
+      @write = write
+      @pid = pid
     end
 
     def stop
@@ -70,7 +82,7 @@ module Parallel
       rescue EOFError
         raise DeadWorker
       end
-      raise result.exception if ExceptionWrapper === result
+      raise result.exception if result.is_a?(ExceptionWrapper)
       result
     end
 
@@ -99,7 +111,7 @@ module Parallel
         item, index = @mutex.synchronize do
           return if @stopped
           item = @lambda.call
-          @stopped = (item == Parallel::Stop)
+          @stopped = (item == Stop)
           return if @stopped
           [item, @index += 1]
         end
@@ -137,7 +149,7 @@ module Parallel
     end
 
     def queue_wrapper(array)
-      array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) }
+      array.respond_to?(:num_waiting) && array.respond_to?(:pop) && -> { array.pop(false) }
     end
   end
 
@@ -153,7 +165,7 @@ module Parallel
 
         if @to_be_killed.empty?
           old_interrupt = trap_interrupt(signal) do
-            $stderr.puts 'Parallel execution interrupted, exiting ...'
+            warn 'Parallel execution interrupted, exiting ...'
             @to_be_killed.flatten.each { |pid| kill(pid) }
           end
         end
@@ -180,7 +192,7 @@ module Parallel
 
         Signal.trap signal do
           yield
-          if old == "DEFAULT"
+          if !old || old == "DEFAULT"
             raise Interrupt
           else
             old.call
@@ -197,37 +209,61 @@ module Parallel
   end
 
   class << self
-    def in_threads(options={:count => 2})
-      count, _ = extract_count_from_options(options)
-      Array.new(count).each_with_index.map do |_, i|
-        Thread.new { yield(i) }
-      end.map!(&:value)
+    def in_threads(options = { count: 2 })
+      threads = []
+      count, = extract_count_from_options(options)
+
+      Thread.handle_interrupt(Exception => :never) do
+        Thread.handle_interrupt(Exception => :immediate) do
+          count.times do |i|
+            threads << Thread.new { yield(i) }
+          end
+          threads.map(&:value)
+        end
+      ensure
+        threads.each(&:kill)
+      end
     end
 
     def in_processes(options = {}, &block)
       count, options = extract_count_from_options(options)
       count ||= processor_count
-      map(0...count, options.merge(:in_processes => count), &block)
+      map(0...count, options.merge(in_processes: count), &block)
+    end
+
+    def each(array, options = {}, &block)
+      map(array, options.merge(preserve_results: false), &block)
     end
 
-    def each(array, options={}, &block)
-      map(array, options.merge(:preserve_results => false), &block)
-      array
+    def any?(*args, &block)
+      raise "You must provide a block when calling #any?" if block.nil?
+      !each(*args) { |*a| raise Kill if block.call(*a) }
     end
 
-    def each_with_index(array, options={}, &block)
-      each(array, options.merge(:with_index => true), &block)
+    def all?(*args, &block)
+      raise "You must provide a block when calling #all?" if block.nil?
+      !!each(*args) { |*a| raise Kill unless block.call(*a) }
+    end
+
+    def each_with_index(array, options = {}, &block)
+      each(array, options.merge(with_index: true), &block)
     end
 
     def map(source, options = {}, &block)
+      options = options.dup
       options[:mutex] = Mutex.new
 
-      if RUBY_PLATFORM =~ /java/ and not options[:in_processes]
+      if options[:in_processes] && options[:in_threads]
+        raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`."
+      elsif RUBY_PLATFORM =~ (/java/) && !(options[:in_processes])
         method = :in_threads
         size = options[method] || processor_count
       elsif options[:in_threads]
         method = :in_threads
         size = options[method]
+      elsif options[:in_ractors]
+        method = :in_ractors
+        size = options[method]
       else
         method = :in_processes
         if Process.respond_to?(:fork)
@@ -244,23 +280,78 @@ module Parallel
       options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
       add_progress_bar!(job_factory, options)
 
-      if size == 0
-        work_direct(job_factory, options, &block)
-      elsif method == :in_threads
-        work_in_threads(job_factory, options.merge(:count => size), &block)
-      else
-        work_in_processes(job_factory, options.merge(:count => size), &block)
+      result =
+        if size == 0
+          work_direct(job_factory, options, &block)
+        elsif method == :in_threads
+          work_in_threads(job_factory, options.merge(count: size), &block)
+        elsif method == :in_ractors
+          work_in_ractors(job_factory, options.merge(count: size), &block)
+        else
+          work_in_processes(job_factory, options.merge(count: size), &block)
+        end
+
+      return result.value if result.is_a?(Break)
+      raise result if result.is_a?(Exception)
+      options[:return_results] ? result : source
+    end
+
+    def map_with_index(array, options = {}, &block)
+      map(array, options.merge(with_index: true), &block)
+    end
+
+    def flat_map(*args, &block)
+      map(*args, &block).flatten(1)
+    end
+
+    def filter_map(*args, &block)
+      map(*args, &block).compact
+    end
+
+    # Number of physical processor cores on the current system.
+    def physical_processor_count
+      @physical_processor_count ||= begin
+        ppc =
+          case RbConfig::CONFIG["target_os"]
+          when /darwin[12]/
+            IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
+          when /linux/
+            cores = {} # unique physical ID / core ID combinations
+            phy = 0
+            File.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
+              if ln.start_with?("physical")
+                phy = ln[/\d+/]
+              elsif ln.start_with?("core")
+                cid = "#{phy}:#{ln[/\d+/]}"
+                cores[cid] = true unless cores[cid]
+              end
+            end
+            cores.count
+          when /mswin|mingw/
+            require 'win32ole'
+            result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
+              "select NumberOfCores from Win32_Processor"
+            )
+            result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
+          else
+            processor_count
+          end
+        # fall back to logical count if physical info is invalid
+        ppc > 0 ? ppc : processor_count
       end
     end
 
-    def map_with_index(array, options={}, &block)
-      map(array, options.merge(:with_index => true), &block)
+    # Number of processors seen by the OS, used for process scheduling
+    def processor_count
+      require 'etc'
+      @processor_count ||= Integer(ENV['PARALLEL_PROCESSOR_COUNT'] || Etc.nprocessors)
     end
 
     def worker_number
       Thread.current[:parallel_worker_number]
     end
 
+    # TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
     def worker_number=(worker_num)
       Thread.current[:parallel_worker_number] = worker_num
     end
@@ -295,13 +386,18 @@ module Parallel
     def work_direct(job_factory, options, &block)
       self.worker_number = 0
       results = []
-      while set = job_factory.next
-        item, index = set
-        results << with_instrumentation(item, index, options) do
-          call_with_index(item, index, options, &block)
+      exception = nil
+      begin
+        while set = job_factory.next
+          item, index = set
+          results << with_instrumentation(item, index, options) do
+            call_with_index(item, index, options, &block)
+          end
         end
+      rescue StandardError
+        exception = $!
       end
-      results
+      exception || results
     ensure
       self.worker_number = nil
     end
@@ -322,21 +418,83 @@ module Parallel
               call_with_index(item, index, options, &block)
             end
             results_mutex.synchronize { results[index] = result }
-          rescue StandardError => e
-            exception = e
+          rescue StandardError
+            exception = $!
           end
         end
       end
 
-      handle_exception(exception, results)
+      exception || results
     end
 
-    def work_in_processes(job_factory, options, &blk)
-      workers = if options[:isolation]
-        [] # we create workers per job and not beforehand
-      else
-        create_workers(job_factory, options, &blk)
+    def work_in_ractors(job_factory, options)
+      exception = nil
+      results = []
+      results_mutex = Mutex.new # arrays are not thread-safe on jRuby
+
+      callback = options[:ractor]
+      if block_given? || !callback
+        raise ArgumentError, "pass the code you want to execute as `ractor: [ClassName, :method_name]`"
       end
+
+      # build
+      ractors = Array.new(options.fetch(:count)) do
+        Ractor.new do
+          loop do
+            got = receive
+            (klass, method_name), item, index = got
+            break if index == :break
+            begin
+              Ractor.yield [nil, klass.send(method_name, item), item, index]
+            rescue StandardError => e
+              Ractor.yield [e, nil, item, index]
+            end
+          end
+        end
+      end
+
+      # start
+      ractors.dup.each do |ractor|
+        if set = job_factory.next
+          item, index = set
+          instrument_start item, index, options
+          ractor.send [callback, item, index]
+        else
+          ractor.send([[nil, nil], nil, :break]) # stop the ractor
+          ractors.delete ractor
+        end
+      end
+
+      # replace with new items
+      while set = job_factory.next
+        item_next, index_next = set
+        done, (exception, result, item, index) = Ractor.select(*ractors)
+        if exception
+          ractors.delete done
+          break
+        end
+        instrument_finish item, index, result, options
+        results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
+
+        instrument_start item_next, index_next, options
+        done.send([callback, item_next, index_next])
+      end
+
+      # finish
+      ractors.each do |ractor|
+        (new_exception, result, item, index) = ractor.take
+        exception ||= new_exception
+        next if new_exception
+        instrument_finish item, index, result, options
+        results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
+        ractor.send([[nil, nil], nil, :break]) # stop the ractor
+      end
+
+      exception || results
+    end
+
+    def work_in_processes(job_factory, options, &blk)
+      workers = create_workers(job_factory, options, &blk)
       results = []
       results_mutex = Mutex.new # arrays are not thread-safe
       exception = nil
@@ -344,6 +502,8 @@ module Parallel
       UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
         in_threads(options) do |i|
           worker = workers[i]
+          worker.thread = Thread.current
+          worked = false
 
           begin
             loop do
@@ -352,11 +512,11 @@ module Parallel
               break unless index
 
               if options[:isolation]
-                worker = replace_worker(job_factory, workers, i, options, blk)
+                worker = replace_worker(job_factory, workers, i, options, blk) if worked
+                worked = true
+                worker.thread = Thread.current
               end
 
-              worker.thread = Thread.current
-
               begin
                 result = with_instrumentation item, index, options do
                   worker.work(job_factory.pack(item, index))
@@ -364,31 +524,33 @@ module Parallel
                 results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
               rescue StandardError => e
                 exception = e
-                if Parallel::Kill === exception
+                if exception.is_a?(Kill)
                   (workers - [worker]).each do |w|
-                    w.thread.kill unless w.thread.nil?
+                    w.thread&.kill
                     UserInterruptHandler.kill(w.pid)
                   end
                 end
               end
             end
           ensure
-            worker.stop if worker
+            worker.stop
           end
         end
       end
 
-      handle_exception(exception, results)
+      exception || results
     end
 
-    def replace_worker(job_factory, workers, i, options, blk)
-      # old worker is no longer used ... stop it
-      worker = workers[i]
-      worker.stop if worker
+    def replace_worker(job_factory, workers, index, options, blk)
+      options[:mutex].synchronize do
+        # old worker is no longer used ... stop it
+        worker = workers[index]
+        worker.stop
 
-      # create a new replacement worker
-      running = workers - [worker]
-      workers[i] = worker(job_factory, options.merge(started_workers: running, worker_number: i), &blk)
+        # create a new replacement worker
+        running = workers - [worker]
+        workers[index] = worker(job_factory, options.merge(started_workers: running, worker_number: index), &blk)
+      end
     end
 
     def create_workers(job_factory, options, &block)
@@ -429,21 +591,25 @@ module Parallel
       until read.eof?
         data = Marshal.load(read)
         item, index = job_factory.unpack(data)
-        result = begin
-          call_with_index(item, index, options, &block)
-        rescue StandardError => e
-          ExceptionWrapper.new(e)
+
+        result =
+          begin
+            call_with_index(item, index, options, &block)
+          # https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
+          rescue NoMemoryError, SignalException, Interrupt, SystemExit # rubocop:disable Lint/ShadowedException
+            raise $!
+          rescue Exception # # rubocop:disable Lint/RescueException
+            ExceptionWrapper.new($!)
+          end
+
+        begin
+          Marshal.dump(result, write)
+        rescue Errno::EPIPE
+          return # parent thread already dead
         end
-        Marshal.dump(result, write)
       end
     end
 
-    def handle_exception(exception, results)
-      return nil if [Parallel::Break, Parallel::Kill].include? exception.class
-      raise exception if exception
-      results
-    end
-
     # options is either a Integer or a Hash with :count
     def extract_count_from_options(options)
       if options.is_a?(Hash)
@@ -458,21 +624,53 @@ module Parallel
     def call_with_index(item, index, options, &block)
       args = [item]
       args << index if options[:with_index]
+      results = block.call(*args)
       if options[:return_results]
-        block.call(*args)
+        results
       else
-        block.call(*args)
         nil # avoid GC overhead of passing large results around
       end
     end
 
     def with_instrumentation(item, index, options)
-      on_start = options[:start]
-      on_finish = options[:finish]
-      options[:mutex].synchronize { on_start.call(item, index) } if on_start
+      instrument_start(item, index, options)
       result = yield
-      options[:mutex].synchronize { on_finish.call(item, index, result) } if on_finish
+      instrument_finish(item, index, result, options)
       result unless options[:preserve_results] == false
     end
+
+    def instrument_finish(item, index, result, options)
+      return unless (on_finish = options[:finish])
+      return instrument_finish_in_order(item, index, result, options) if options[:finish_in_order]
+      options[:mutex].synchronize { on_finish.call(item, index, result) }
+    end
+
+    # yield results in the order of the input items
+    # needs to use `options` to store state between executions
+    # needs to use `done` index since a nil result would also be valid
+    def instrument_finish_in_order(item, index, result, options)
+      options[:mutex].synchronize do
+        # initialize our state
+        options[:finish_done] ||= []
+        options[:finish_expecting] ||= 0 # we wait for item at index 0
+
+        # store current result
+        options[:finish_done][index] = [item, result]
+
+        # yield all results that are now in order
+        break unless index == options[:finish_expecting]
+        index.upto(options[:finish_done].size).each do |i|
+          break unless (done = options[:finish_done][i])
+          options[:finish_done][i] = nil # allow GC to free this item and result
+          options[:finish].call(done[0], i, done[1])
+          options[:finish_expecting] += 1
+        end
+      end
+    end
+
+    def instrument_start(item, index, options)
+      return unless on_start = options[:start]
+      options[:mutex].synchronize { on_start.call(item, index) }
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parallel
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.24.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-24 00:00:00.000000000 Z
+date: 2023-12-16 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: michael@grosser.it
@@ -18,12 +18,15 @@ extra_rdoc_files: []
 files:
 - MIT-LICENSE.txt
 - lib/parallel.rb
-- lib/parallel/processor_count.rb
 - lib/parallel/version.rb
 homepage: https://github.com/grosser/parallel
 licenses:
 - MIT
-metadata: {}
+metadata:
+  bug_tracker_uri: https://github.com/grosser/parallel/issues
+  documentation_uri: https://github.com/grosser/parallel/blob/v1.24.0/Readme.md
+  source_code_uri: https://github.com/grosser/parallel/tree/v1.24.0
+  wiki_uri: https://github.com/grosser/parallel/wiki
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -32,15 +35,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Run any kind of code in parallel processes

data/lib/parallel/processor_count.rb

@@ -1,85 +0,0 @@
-module Parallel
-  module ProcessorCount
-    # Number of processors seen by the OS and used for process scheduling.
-    #
-    # * AIX: /usr/sbin/pmcycles (AIX 5+), /usr/sbin/lsdev
-    # * BSD: /sbin/sysctl
-    # * Cygwin: /proc/cpuinfo
-    # * Darwin: /usr/bin/hwprefs, /usr/sbin/sysctl
-    # * HP-UX: /usr/sbin/ioscan
-    # * IRIX: /usr/sbin/sysconf
-    # * Linux: /proc/cpuinfo
-    # * Minix 3+: /proc/cpuinfo
-    # * Solaris: /usr/sbin/psrinfo
-    # * Tru64 UNIX: /usr/sbin/psrinfo
-    # * UnixWare: /usr/sbin/psrinfo
-    #
-    def processor_count
-      @processor_count ||= begin
-        os_name = RbConfig::CONFIG["target_os"]
-        if os_name =~ /mingw|mswin/
-          require 'win32ole'
-          result = WIN32OLE.connect("winmgmts://").ExecQuery(
-            "select NumberOfLogicalProcessors from Win32_Processor")
-          result.to_enum.collect(&:NumberOfLogicalProcessors).reduce(:+)
-        elsif File.readable?("/proc/cpuinfo")
-          IO.read("/proc/cpuinfo").scan(/^processor/).size
-        elsif File.executable?("/usr/bin/hwprefs")
-          IO.popen("/usr/bin/hwprefs thread_count").read.to_i
-        elsif File.executable?("/usr/sbin/psrinfo")
-          IO.popen("/usr/sbin/psrinfo").read.scan(/^.*on-*line/).size
-        elsif File.executable?("/usr/sbin/ioscan")
-          IO.popen("/usr/sbin/ioscan -kC processor") do |out|
-            out.read.scan(/^.*processor/).size
-          end
-        elsif File.executable?("/usr/sbin/pmcycles")
-          IO.popen("/usr/sbin/pmcycles -m").read.count("\n")
-        elsif File.executable?("/usr/sbin/lsdev")
-          IO.popen("/usr/sbin/lsdev -Cc processor -S 1").read.count("\n")
-        elsif File.executable?("/usr/sbin/sysconf") and os_name =~ /irix/i
-          IO.popen("/usr/sbin/sysconf NPROC_ONLN").read.to_i
-        elsif File.executable?("/usr/sbin/sysctl")
-          IO.popen("/usr/sbin/sysctl -n hw.ncpu").read.to_i
-        elsif File.executable?("/sbin/sysctl")
-          IO.popen("/sbin/sysctl -n hw.ncpu").read.to_i
-        else
-          $stderr.puts "Unknown platform: " + RbConfig::CONFIG["target_os"]
-          $stderr.puts "Assuming 1 processor."
-          1
-        end
-      end
-    end
-
-    # Number of physical processor cores on the current system.
-    #
-    def physical_processor_count
-      @physical_processor_count ||= begin
-        ppc = case RbConfig::CONFIG["target_os"]
-        when /darwin1/
-          IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
-        when /linux/
-          cores = {}  # unique physical ID / core ID combinations
-          phy = 0
-          IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
-            if ln.start_with?("physical")
-              phy = ln[/\d+/]
-            elsif ln.start_with?("core")
-              cid = phy + ":" + ln[/\d+/]
-              cores[cid] = true if not cores[cid]
-            end
-          end
-          cores.count
-        when /mswin|mingw/
-          require 'win32ole'
-          result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
-            "select NumberOfCores from Win32_Processor")
-          result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
-        else
-          processor_count
-        end
-        # fall back to logical count if physical info is invalid
-        ppc > 0 ? ppc : processor_count
-      end
-    end
-  end
-end