parallel 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3331bc6634cd376e4f3f8511049e49a0d4501110aaeac395e2f0c0b638f4a402
-  data.tar.gz: 357e1424cb8297b6472c2c5b9486e26da3969edd928be7d6ca7f21670211a8e0
+  metadata.gz: bcd30f5bfe7076b4a0872b389f00057efaae80917298e3d227c160cc8c1be410
+  data.tar.gz: d6dd66ee7f75d2e14123c292ce13ac2b448134622c712650c07482986dbd48a3
 SHA512:
-  metadata.gz: 1885d4f814023905f76105f0edce9155024478408be851a4f6869e537400e924e03d34196e03b428803c274b5fbf7e9bf2d93f4548782aa6d3015063c7ed7883
-  data.tar.gz: 97d22f6b0320a089a3584e76c0baa245811861776e7337e913961cd64cfe0770131493d7de7e1355a62d6339d47d0b1f3ad32dd89084f31c69a61fcfab9f3148
+  metadata.gz: 11ed4f49a53e9e20d052725911593266b808dfaa7b2392f503d39494b318a16df15ad628b586f93b700f5e22627116c465afff818d61ac2f0ca3aed4dcd1b8c0
+  data.tar.gz: af37003c9dc58ae81dac2a80355f5b16db9f54734f06a9c0dc33e73730818f7e81306237f152cd2976399a6453300915b19593e2e6928b480330fa736c516bc6

data/lib/parallel/serializer.rb

@@ -24,6 +24,10 @@ module Parallel
         @secret = secret
       end
 
+      def inspect
+        "#<#{self.class} @inner=#{@inner.inspect}, @secret=[REDACTED]>"
+      end
+
       def dump(data, io)
         payload = @inner.dump(data)
         mac = OpenSSL::HMAC.digest('SHA256', @secret, payload)

data/lib/parallel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Parallel
-  VERSION = Version = '2.1.0' # rubocop:disable Naming/ConstantName
+  VERSION = Version = '2.2.0' # rubocop:disable Naming/ConstantName
 end

data/lib/parallel.rb

@@ -124,7 +124,7 @@ module Parallel
         item, index = @mutex.synchronize do
           return if @stopped
           item = @lambda.call
-          @stopped = (item == Stop)
+          @stopped = Stop.equal?(item)
           return if @stopped
           [item, @index += 1]
         end
@@ -178,7 +178,7 @@ module Parallel
 
         if @to_be_killed.empty?
           old_interrupt = trap_interrupt(signal) do
-            warn 'Parallel execution interrupted, exiting ...'
+            warn 'parallel: execution interrupted, exiting ...'
             @to_be_killed.flatten.each { |pid| kill(pid) }
           end
         end
@@ -245,7 +245,7 @@ module Parallel
     end
 
     def each(array, options = {}, &block)
-      map(array, options.merge(preserve_results: false), &block)
+      map(array, options.merge(discard_results: true), &block)
     end
 
     def any?(*args, &block)
@@ -266,9 +266,15 @@ module Parallel
       options = options.dup
       options[:mutex] = Mutex.new
 
-      if options[:in_processes] && options[:in_threads]
-        raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`."
-      elsif RUBY_PLATFORM.include?('java') && !options[:in_processes]
+      if options.slice(:in_processes, :in_threads, :in_ractors).size > 1
+        raise ArgumentError, "Use only one of `in_processes`, `in_threads`, or `in_ractors`."
+      end
+
+      if options[:in_ractors] ? block : options[:ractor]
+        raise ArgumentError, "use either in_ractors with :ractor or a not in_ractors and block"
+      end
+
+      if RUBY_PLATFORM.include?('java') && !options[:in_processes]
         method = :in_threads
         size = options[method] || processor_count
       elsif options[:in_threads]
@@ -282,31 +288,38 @@ module Parallel
         if Process.respond_to?(:fork)
           size = options[method] || processor_count
         else
-          warn "Process.fork is not supported by this Ruby"
+          warn "parallel: Process.fork is not supported by this Ruby"
           size = 0
         end
       end
 
+      raise ArgumentError, "worker count must be a non-negative Integer" unless size.is_a?(Integer) && size >= 0
+
       job_factory = JobFactory.new(source, options[:mutex])
       size = [job_factory.size, size].min
 
-      options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
+      discard_results = options[:discard_results]
+
+      # finish callback needs the results, careful to do that before add_progress_bar which adds finish
+      options[:discard_results] = discard_results && !options[:finish]
+
       add_progress_bar!(job_factory, options)
 
       result =
         if size == 0
+          block = ractor_block(options) if method == :in_ractors
           work_direct(job_factory, options, &block)
         elsif method == :in_threads
           work_in_threads(job_factory, options.merge(count: size), &block)
         elsif method == :in_ractors
-          work_in_ractors(job_factory, options.merge(count: size), &block)
+          work_in_ractors(job_factory, options.merge(count: size))
         else
           work_in_processes(job_factory, options.merge(count: size), &block)
         end
 
       return result.value if result.is_a?(Break)
       raise result if result.is_a?(Exception)
-      options[:return_results] ? result : source
+      discard_results ? source : result
     end
 
     def map_with_index(array, options = {}, &block)
@@ -318,7 +331,7 @@ module Parallel
     end
 
     def filter_map(...)
-      map(...).compact
+      map(...).select { |value| value }
     end
 
     # Number of physical processor cores on the current system.
@@ -379,7 +392,7 @@ module Parallel
       end
       if !result || $?.exitstatus != 0
         # Bail out if both commands returned something unexpected
-        warn "guessing pyhsical processor count"
+        warn "parallel: guessing physical processor count"
         processor_count
       else
         # powershell: "\nNumberOfCores\n-------------\n            4\n\n\n"
@@ -420,22 +433,24 @@ module Parallel
     end
 
     def work_direct(job_factory, options, &block)
+      previous_worker_number = worker_number
       self.worker_number = 0
       results = []
       exception = nil
       begin
         while (set = job_factory.next)
           item, index = set
-          results << with_instrumentation(item, index, options) do
+          result = with_instrumentation(item, index, options) do
             call_with_index(item, index, options, &block)
           end
+          results << result unless options[:discard_results]
         end
       rescue StandardError
         exception = $!
       end
       exception || results
     ensure
-      self.worker_number = nil
+      self.worker_number = previous_worker_number
     end
 
     def work_in_threads(job_factory, options, &block)
@@ -453,7 +468,7 @@ module Parallel
             result = with_instrumentation item, index, options do
               call_with_index(item, index, options, &block)
             end
-            results_mutex.synchronize { results[index] = result }
+            results_mutex.synchronize { results[index] = result } unless options[:discard_results]
           rescue StandardError
             exception = $!
           end
@@ -482,55 +497,70 @@ module Parallel
         ports[port] = ractor
       end
 
-      # start
-      ports.dup.each do |port, ractor|
-        if (job = job_factory.next)
-          item, index = job
-          instrument_start item, index, options
-          ractor.send [callback, item, index]
-        else # not enough work, `receive` would hang
-          ractor_stop ractor
-          ports.delete port
+      begin
+        # start by sending 1 item each
+        ports.dup.each do |port, ractor|
+          if (job = job_factory.next)
+            ractor_send ractor, callback, job, options
+          else # not enough work, `receive` would hang
+            ractor_stop ractor
+            ports.delete port
+          end
         end
-      end
 
-      # receive result and send new items to done ractors
-      while (job = job_factory.next)
-        # receive result
-        done_port, (exception, result, item_prev, index_prev) = Ractor.select(*ports.keys)
-        done_ractor = ports[done_port]
-        if exception
-          ports.delete done_port
-          break
-        end
-        ractor_result item_prev, index_prev, result, results, results_mutex, options
+        # receive result and send new items
+        while (job = job_factory.next)
+          # receive result
+          done_port, (exception, result, item_prev, index_prev) = Ractor.select(*ports.keys)
+          done_ractor = ports[done_port]
+          if exception
+            ractor_stop done_ractor
+            ports.delete done_port
+            break
+          end
+          ractor_result item_prev, index_prev, result, results, results_mutex, options
 
-        # send new
-        item_next, index_next = job
-        instrument_start item_next, index_next, options
-        done_ractor.send([callback, item_next, index_next])
-      end
+          # send new
+          ractor_send done_ractor, callback, job, options
+        end
 
-      # finish
-      ports.each do |port, ractor|
-        (new_exception, result, item, index) = use_port ? port.receive : ractor.take
-        exception ||= new_exception
-        next if new_exception
-        ractor_result item, index, result, results, results_mutex, options
-        ractor_stop ractor
+        # finish by receiving the last results
+        ports.dup.each do |port, ractor|
+          (new_exception, result, item, index) = use_port ? port.receive : ractor.take
+          exception ||= new_exception
+          ractor_result item, index, result, results, results_mutex, options unless new_exception
+          ractor_stop ractor
+          ports.delete port
+        end
+      ensure # close any ports that remained open in case something blew up
+        ports.each_value do |ractor|
+          ractor_stop(ractor)
+          ractor.take unless use_port
+        rescue Ractor::ClosedError, Ractor::RemoteError
+          nil
+        end
       end
 
       exception || results
     end
 
+    # ractors cannot execute blocks, so run the callback directly when not using ractors
+    def ractor_block(options)
+      klass, method_name = options[:ractor] ||
+        raise(ArgumentError, "pass the code you want to execute as `ractor: [ClassName, :method_name]`")
+      ->(*args) { klass.send(method_name, *args) }
+    end
+
     def ractor_build(use_port)
       args = use_port ? [Ractor::Port.new] : []
       ractor = Ractor.new(*args) do |port|
         loop do
-          (klass, method_name), item, index = receive
+          (klass, method_name), item, index, with_index, discard_results = receive
           break if index == :break
           begin
-            result = [nil, klass.send(method_name, item), item, index]
+            value = with_index ? klass.send(method_name, item, index) : klass.send(method_name, item)
+            value = nil if discard_results
+            result = [nil, value, item, index]
           rescue StandardError => e
             result = [e, nil, item, index]
           end
@@ -546,11 +576,19 @@ module Parallel
 
     def ractor_result(item, index, result, results, results_mutex, options)
       instrument_finish item, index, result, options
-      results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
+      results_mutex.synchronize { results[index] = result } unless options[:discard_results]
+    end
+
+    def ractor_send(ractor, callback, job, options)
+      item, index = job
+      instrument_start item, index, options
+      ractor.send [callback, item, index, options[:with_index], options[:discard_results]]
     end
 
     def ractor_stop(ractor)
       ractor.send([[nil, nil], nil, :break])
+    rescue Ractor::ClosedError
+      nil
     end
 
     def work_in_processes(job_factory, options, &blk)
@@ -581,7 +619,8 @@ module Parallel
                 result = with_instrumentation item, index, options do
                   worker.work(job_factory.pack(item, index))
                 end
-                results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
+                # arrays are not threads safe on jRuby
+                results_mutex.synchronize { results[index] = result } unless options[:discard_results]
               rescue StandardError => e
                 exception = e
                 if exception.is_a?(Kill)
@@ -619,6 +658,13 @@ module Parallel
         workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block)
       end
       workers
+    rescue Exception # rubocop:disable Lint/RescueException
+      workers.each do |worker|
+        worker.stop
+      rescue StandardError
+        nil
+      end
+      raise
     end
 
     def worker(job_factory, options, &block)
@@ -646,6 +692,19 @@ module Parallel
       child_write.close
 
       Worker.new(parent_read, parent_write, pid, options[:serializer])
+    rescue Exception # rubocop:disable Lint/RescueException
+      [child_read, parent_write, parent_read, child_write].compact.each do |io|
+        io.close unless io.closed?
+      end
+      if pid
+        UserInterruptHandler.kill(pid)
+        begin
+          Process.wait(pid)
+        rescue Errno::ECHILD
+          nil
+        end
+      end
+      raise
     end
 
     def process_incoming_jobs(read, write, job_factory, options, &block)
@@ -687,10 +746,10 @@ module Parallel
       args = [item]
       args << index if options[:with_index]
       results = block.call(*args)
-      if options[:return_results]
-        results
-      else
+      if options[:discard_results]
         nil # avoid GC overhead of passing large results around
+      else
+        results
       end
     end
 
@@ -698,7 +757,7 @@ module Parallel
       instrument_start(item, index, options)
       result = yield
       instrument_finish(item, index, result, options)
-      result unless options[:preserve_results] == false
+      result unless options[:discard_results]
     end
 
     def instrument_finish(item, index, result, options)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parallel
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Michael Grosser
@@ -23,10 +23,10 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/grosser/parallel/issues
-  documentation_uri: https://github.com/grosser/parallel/blob/v2.1.0/Readme.md
-  source_code_uri: https://github.com/grosser/parallel/tree/v2.1.0
+  documentation_uri: https://github.com/grosser/parallel/blob/v2.2.0/Readme.md
+  source_code_uri: https://github.com/grosser/parallel/tree/v2.2.0
   wiki_uri: https://github.com/grosser/parallel/wiki
-  changelog_uri: https://github.com/grosser/parallel/blob/v2.1.0/CHANGELOG.md
+  changelog_uri: https://github.com/grosser/parallel/blob/v2.2.0/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: