parallel 1.13.0 → 1.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89d859597b00942e642e88797290b1407a87615c0bb9756b8a73a3c976410aac
-  data.tar.gz: 40537d9251999345344d79a6ef06946257301af2ddbe5484e1c97e6dea28d356
+  metadata.gz: fa334cb0d83b8049f260a5e6724d2fd0e65768f85afd0968a2af05d55dda4e0e
+  data.tar.gz: 7ba3ade3a7af4bcfe5944b8d7313b5ec6e4074d27755a83f95e4f8a01d77261e
 SHA512:
-  metadata.gz: 45a8ae3ecf5ded1f1c53aaf98cc5bb45b6e1c60e51997143a2420433f4a94d0de4e339a6a9be8f276432f8bdff359fdeb10fdc5e8aa10e34874973c5c84384c4
-  data.tar.gz: 9b2a309db6cd24a199fa2fd99279b4015df984c09a65896e6a56a11f080d691fceabb38684606f2c2e3291794ea50cac764c82fbb93d7c32a22720b93960c2e4
+  metadata.gz: d470562bd90ab80c66f69b9c1ab016f737bf52e091ac3027c51ea7482eaecd3bd3c186daab756c5d4276b9fb54f0e80f149a8071dc17c31a321ff4fc4d369600
+  data.tar.gz: f01bf1c3a052579c11c3097b02b02a4a30c8e881dbb4916c3ceba6f122aab3a40c276a1de82f7ea7d3790eeff2f994bd20283361151439a3d57b025bdd24dfc8

data/lib/parallel.rb

@@ -3,15 +3,21 @@ require 'parallel/version'
 require 'parallel/processor_count'
 
 module Parallel
-  extend Parallel::ProcessorCount
+  extend ProcessorCount
+
+  Stop = Object.new.freeze
 
   class DeadWorker < StandardError
   end
 
   class Break < StandardError
+    attr_reader :value
+    def initialize(value = nil)
+      @value = value
+    end
   end
 
-  class Kill < StandardError
+  class Kill < Break
   end
 
   class UndumpableException < StandardError
@@ -22,8 +28,6 @@ module Parallel
     end
   end
 
-  Stop = Object.new
-
   class ExceptionWrapper
     attr_reader :exception
     def initialize(exception)
@@ -102,7 +106,7 @@ module Parallel
         item, index = @mutex.synchronize do
           return if @stopped
           item = @lambda.call
-          @stopped = (item == Parallel::Stop)
+          @stopped = (item == Stop)
           return if @stopped
           [item, @index += 1]
         end
@@ -201,10 +205,21 @@ module Parallel
 
   class << self
     def in_threads(options={:count => 2})
+      threads = []
       count, _ = extract_count_from_options(options)
-      Array.new(count) do |i|
-        Thread.new { yield(i) }
-      end.map!(&:value)
+
+      Thread.handle_interrupt(Exception => :never) do
+        begin
+          Thread.handle_interrupt(Exception => :immediate) do
+            count.times do |i|
+              threads << Thread.new { yield(i) }
+            end
+            threads.map(&:value)
+          end
+        ensure
+          threads.each(&:kill)
+        end
+      end
     end
 
     def in_processes(options = {}, &block)
@@ -219,12 +234,12 @@ module Parallel
 
     def any?(*args, &block)
       raise "You must provide a block when calling #any?" if block.nil?
-      !each(*args) { |*a| raise Parallel::Kill if block.call(*a) }
+      !each(*args) { |*a| raise Kill if block.call(*a) }
     end
 
     def all?(*args, &block)
       raise "You must provide a block when calling #all?" if block.nil?
-      !!each(*args) { |*a| raise Parallel::Kill unless block.call(*a) }
+      !!each(*args) { |*a| raise Kill unless block.call(*a) }
     end
 
     def each_with_index(array, options={}, &block)
@@ -259,26 +274,33 @@ module Parallel
       options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
       add_progress_bar!(job_factory, options)
 
-      results = if size == 0
-        work_direct(job_factory, options, &block)
-      elsif method == :in_threads
-        work_in_threads(job_factory, options.merge(:count => size), &block)
-      else
-        work_in_processes(job_factory, options.merge(:count => size), &block)
-      end
-      if results
-        options[:return_results] ? results : source
-      end
+      result =
+        if size == 0
+          work_direct(job_factory, options, &block)
+        elsif method == :in_threads
+          work_in_threads(job_factory, options.merge(:count => size), &block)
+        else
+          work_in_processes(job_factory, options.merge(:count => size), &block)
+        end
+
+      return result.value if result.is_a?(Break)
+      raise result if result.is_a?(Exception)
+      options[:return_results] ? result : source
     end
 
     def map_with_index(array, options={}, &block)
       map(array, options.merge(:with_index => true), &block)
     end
 
+    def flat_map(*args, &block)
+      map(*args, &block).flatten(1)
+    end
+
     def worker_number
       Thread.current[:parallel_worker_number]
     end
 
+    # TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
     def worker_number=(worker_num)
       Thread.current[:parallel_worker_number] = worker_num
     end
@@ -324,7 +346,7 @@ module Parallel
       rescue
         exception = $!
       end
-      handle_exception(exception, results)
+      exception || results
     ensure
       self.worker_number = nil
     end
@@ -351,15 +373,11 @@ module Parallel
         end
       end
 
-      handle_exception(exception, results)
+      exception || results
     end
 
     def work_in_processes(job_factory, options, &blk)
-      workers = if options[:isolation]
-        [] # we create workers per job and not beforehand
-      else
-        create_workers(job_factory, options, &blk)
-      end
+      workers = create_workers(job_factory, options, &blk)
       results = []
       results_mutex = Mutex.new # arrays are not thread-safe
       exception = nil
@@ -367,6 +385,8 @@ module Parallel
       UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
         in_threads(options) do |i|
           worker = workers[i]
+          worker.thread = Thread.current
+          worked = false
 
           begin
             loop do
@@ -375,40 +395,39 @@ module Parallel
               break unless index
 
               if options[:isolation]
-                worker = replace_worker(job_factory, workers, i, options, blk)
+                worker = replace_worker(job_factory, workers, i, options, blk) if worked
+                worked = true
+                worker.thread = Thread.current
               end
 
-              worker.thread = Thread.current
-
               begin
                 result = with_instrumentation item, index, options do
                   worker.work(job_factory.pack(item, index))
                 end
                 results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
-              rescue
-                exception = $!
-                if Parallel::Kill === exception
+              rescue StandardError => e
+                exception = e
+                if Kill === exception
                   (workers - [worker]).each do |w|
-                    w.thread.kill unless w.thread.nil?
+                    w.thread.kill if w.thread
                     UserInterruptHandler.kill(w.pid)
                   end
                 end
               end
             end
           ensure
-            worker.stop if worker
+            worker.stop
           end
         end
       end
-
-      handle_exception(exception, results)
+      exception || results
     end
 
     def replace_worker(job_factory, workers, i, options, blk)
       options[:mutex].synchronize do
         # old worker is no longer used ... stop it
         worker = workers[i]
-        worker.stop if worker
+        worker.stop
 
         # create a new replacement worker
         running = workers - [worker]
@@ -456,19 +475,20 @@ module Parallel
         item, index = job_factory.unpack(data)
         result = begin
           call_with_index(item, index, options, &block)
-        rescue
+        # https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
+        rescue NoMemoryError, SignalException, Interrupt, SystemExit
+          raise $!
+        rescue Exception
           ExceptionWrapper.new($!)
         end
-        Marshal.dump(result, write)
+        begin
+          Marshal.dump(result, write)
+        rescue Errno::EPIPE
+          return # parent thread already dead
+        end
       end
     end
 
-    def handle_exception(exception, results)
-      return nil if [Parallel::Break, Parallel::Kill].include? exception.class
-      raise exception if exception
-      results
-    end
-
     # options is either a Integer or a Hash with :count
     def extract_count_from_options(options)
       if options.is_a?(Hash)

data/lib/parallel/processor_count.rb

@@ -1,65 +1,14 @@
-if RUBY_VERSION.to_f >= 2.2
-  require 'etc'
-end
+require 'etc'
 
 module Parallel
+  # TODO: inline this method into parallel.rb and kill physical_processor_count in next major release
   module ProcessorCount
-    # Number of processors seen by the OS and used for process scheduling.
-    #
-    # * AIX: /usr/sbin/pmcycles (AIX 5+), /usr/sbin/lsdev
-    # * BSD: /sbin/sysctl
-    # * Cygwin: /proc/cpuinfo
-    # * Darwin: /usr/bin/hwprefs, /usr/sbin/sysctl
-    # * HP-UX: /usr/sbin/ioscan
-    # * IRIX: /usr/sbin/sysconf
-    # * Linux: /proc/cpuinfo
-    # * Minix 3+: /proc/cpuinfo
-    # * Solaris: /usr/sbin/psrinfo
-    # * Tru64 UNIX: /usr/sbin/psrinfo
-    # * UnixWare: /usr/sbin/psrinfo
-    #
+    # Number of processors seen by the OS, used for process scheduling
     def processor_count
-      @processor_count ||= begin
-        if defined?(Etc) && Etc.respond_to?(:nprocessors)
-          Etc.nprocessors
-        else
-          os_name = RbConfig::CONFIG["target_os"]
-          if os_name =~ /mingw|mswin/
-            require 'win32ole'
-            result = WIN32OLE.connect("winmgmts://").ExecQuery(
-              "select NumberOfLogicalProcessors from Win32_Processor")
-            result.to_enum.collect(&:NumberOfLogicalProcessors).reduce(:+)
-          elsif File.readable?("/proc/cpuinfo")
-            IO.read("/proc/cpuinfo").scan(/^processor/).size
-          elsif File.executable?("/usr/bin/hwprefs")
-            IO.popen("/usr/bin/hwprefs thread_count").read.to_i
-          elsif File.executable?("/usr/sbin/psrinfo")
-            IO.popen("/usr/sbin/psrinfo").read.scan(/^.*on-*line/).size
-          elsif File.executable?("/usr/sbin/ioscan")
-            IO.popen("/usr/sbin/ioscan -kC processor") do |out|
-              out.read.scan(/^.*processor/).size
-            end
-          elsif File.executable?("/usr/sbin/pmcycles")
-            IO.popen("/usr/sbin/pmcycles -m").read.count("\n")
-          elsif File.executable?("/usr/sbin/lsdev")
-            IO.popen("/usr/sbin/lsdev -Cc processor -S 1").read.count("\n")
-          elsif File.executable?("/usr/sbin/sysconf") and os_name =~ /irix/i
-            IO.popen("/usr/sbin/sysconf NPROC_ONLN").read.to_i
-          elsif File.executable?("/usr/sbin/sysctl")
-            IO.popen("/usr/sbin/sysctl -n hw.ncpu").read.to_i
-          elsif File.executable?("/sbin/sysctl")
-            IO.popen("/sbin/sysctl -n hw.ncpu").read.to_i
-          else
-            $stderr.puts "Unknown platform: " + RbConfig::CONFIG["target_os"]
-            $stderr.puts "Assuming 1 processor."
-            1
-          end
-        end
-      end
+      @processor_count ||= Integer(ENV['PARALLEL_PROCESSOR_COUNT'] || Etc.nprocessors)
     end
 
     # Number of physical processor cores on the current system.
-    #
     def physical_processor_count
       @physical_processor_count ||= begin
         ppc = case RbConfig::CONFIG["target_os"]

data/lib/parallel/version.rb

@@ -1,3 +1,3 @@
 module Parallel
-  VERSION = Version = '1.13.0'
+  VERSION = Version = '1.20.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parallel
 version: !ruby/object:Gem::Version
-  version: 1.13.0
+  version: 1.20.1
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-17 00:00:00.000000000 Z
+date: 2020-11-22 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: michael@grosser.it
@@ -23,7 +23,11 @@ files:
 homepage: https://github.com/grosser/parallel
 licenses:
 - MIT
-metadata: {}
+metadata:
+  bug_tracker_uri: https://github.com/grosser/parallel/issues
+  documentation_uri: https://github.com/grosser/parallel/blob/v1.20.1/Readme.md
+  source_code_uri: https://github.com/grosser/parallel/tree/v1.20.1
+  wiki_uri: https://github.com/grosser/parallel/wiki
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -32,15 +36,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.3
 signing_key: 
 specification_version: 4
 summary: Run any kind of code in parallel processes