parallel 2.0.1 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0556b829a420b259f2c4610988421e0b21516101042373b14cb9acfe870e9c8d
-  data.tar.gz: 6c0a7aa158fb4a905902c1ef94e3901b75c73518754faa57a421cea4e82df6c7
+  metadata.gz: bcd30f5bfe7076b4a0872b389f00057efaae80917298e3d227c160cc8c1be410
+  data.tar.gz: d6dd66ee7f75d2e14123c292ce13ac2b448134622c712650c07482986dbd48a3
 SHA512:
-  metadata.gz: 76173acf1c6a08e53bf12cfe4c039b02ca5d891b7a0bfed7024a2ba0093d7cbe42790f6e2ac1ee8e195c99e3582c24fb2abb27c8882541361bdc794492df86f3
-  data.tar.gz: a032ace70ba955047cd3d1bb73c85af8bf67664f41e2d61ccdc0d1298277754a8ce190e1f33d654507236a1208d532e7fe973cfe5abee55abd46f7b13b70be36
+  metadata.gz: 11ed4f49a53e9e20d052725911593266b808dfaa7b2392f503d39494b318a16df15ad628b586f93b700f5e22627116c465afff818d61ac2f0ca3aed4dcd1b8c0
+  data.tar.gz: af37003c9dc58ae81dac2a80355f5b16db9f54734f06a9c0dc33e73730818f7e81306237f152cd2976399a6453300915b19593e2e6928b480330fa736c516bc6

data/lib/parallel/serializer.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+require 'openssl'
+require 'securerandom'
+
+module Parallel
+  # Pluggable wire serializers. Each must respond to `dump(data, io)` /
+  # `load(io)` (used directly by Worker) and `dump(data)` / `load(string)`
+  # (used by wrappers like Hmac).
+  module Serializer
+    # Raw Marshal. Fast but trusts anything written to the pipe — a same-UID
+    # attacker that reopens /proc/<pid>/fd/<n> can inject Marshal gadgets (RCE).
+    Marshal = ::Marshal
+
+    # Wraps any inner serializer with a length-prefixed HMAC-SHA256 frame keyed
+    # on a per-worker secret generated before fork. Forged frames from a
+    # pipe-injector fail verification.
+    class Hmac
+      LENGTH_FORMAT = 'N' # 32-bit big-endian unsigned int
+      LENGTH_BYTES = 4
+      MAC_BYTES = 32 # SHA256
+
+      def initialize(inner: Marshal, secret: SecureRandom.bytes(32))
+        @inner = inner
+        @secret = secret
+      end
+
+      def inspect
+        "#<#{self.class} @inner=#{@inner.inspect}, @secret=[REDACTED]>"
+      end
+
+      def dump(data, io)
+        payload = @inner.dump(data)
+        mac = OpenSSL::HMAC.digest('SHA256', @secret, payload)
+        io.write([payload.bytesize].pack(LENGTH_FORMAT), mac, payload)
+      end
+
+      def load(io)
+        # nil at frame boundary = clean EOF (worker died / pipe closed between messages)
+        header = io.read(LENGTH_BYTES) || raise(EOFError) # eof stops worker
+        raise SecurityError, "truncated frame header" if header.bytesize != LENGTH_BYTES
+
+        length = header.unpack1(LENGTH_FORMAT)
+        mac = io.read(MAC_BYTES)
+        raise SecurityError, "truncated frame mac" if mac.nil? || mac.bytesize != MAC_BYTES
+
+        payload = io.read(length)
+        raise SecurityError, "truncated frame payload" if payload.nil? || payload.bytesize != length
+
+        expected = OpenSSL::HMAC.digest('SHA256', @secret, payload)
+        raise SecurityError, "HMAC mismatch on worker pipe" unless OpenSSL.fixed_length_secure_compare(mac, expected)
+
+        @inner.load(payload)
+      end
+    end
+  end
+end

data/lib/parallel/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Parallel
-  VERSION = Version = '2.0.1' # rubocop:disable Naming/ConstantName
+  VERSION = Version = '2.2.0' # rubocop:disable Naming/ConstantName
 end

data/lib/parallel.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'rbconfig'
 require 'parallel/version'
+require 'parallel/serializer'
 
 module Parallel
   Stop = Object.new.freeze
@@ -63,10 +64,11 @@ module Parallel
     attr_reader :pid, :read, :write
     attr_accessor :thread
 
-    def initialize(read, write, pid)
+    def initialize(read, write, pid, serializer)
       @read = read
       @write = write
       @pid = pid
+      @serializer = serializer
     end
 
     def stop
@@ -83,13 +85,13 @@ module Parallel
 
     def work(data)
       begin
-        Marshal.dump(data, write)
+        @serializer.dump(data, write)
       rescue Errno::EPIPE
         raise DeadWorker
       end
 
       result = begin
-        Marshal.load(read)
+        @serializer.load(read)
       rescue EOFError
         raise DeadWorker
       end
@@ -122,7 +124,7 @@ module Parallel
         item, index = @mutex.synchronize do
           return if @stopped
           item = @lambda.call
-          @stopped = (item == Stop)
+          @stopped = Stop.equal?(item)
           return if @stopped
           [item, @index += 1]
         end
@@ -176,7 +178,7 @@ module Parallel
 
         if @to_be_killed.empty?
           old_interrupt = trap_interrupt(signal) do
-            warn 'Parallel execution interrupted, exiting ...'
+            warn 'parallel: execution interrupted, exiting ...'
             @to_be_killed.flatten.each { |pid| kill(pid) }
           end
         end
@@ -243,7 +245,7 @@ module Parallel
     end
 
     def each(array, options = {}, &block)
-      map(array, options.merge(preserve_results: false), &block)
+      map(array, options.merge(discard_results: true), &block)
     end
 
     def any?(*args, &block)
@@ -264,9 +266,15 @@ module Parallel
       options = options.dup
       options[:mutex] = Mutex.new
 
-      if options[:in_processes] && options[:in_threads]
-        raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`."
-      elsif RUBY_PLATFORM =~ /java/ && !options[:in_processes]
+      if options.slice(:in_processes, :in_threads, :in_ractors).size > 1
+        raise ArgumentError, "Use only one of `in_processes`, `in_threads`, or `in_ractors`."
+      end
+
+      if options[:in_ractors] ? block : options[:ractor]
+        raise ArgumentError, "use either in_ractors with :ractor or a not in_ractors and block"
+      end
+
+      if RUBY_PLATFORM.include?('java') && !options[:in_processes]
         method = :in_threads
         size = options[method] || processor_count
       elsif options[:in_threads]
@@ -280,31 +288,38 @@ module Parallel
         if Process.respond_to?(:fork)
           size = options[method] || processor_count
         else
-          warn "Process.fork is not supported by this Ruby"
+          warn "parallel: Process.fork is not supported by this Ruby"
           size = 0
         end
       end
 
+      raise ArgumentError, "worker count must be a non-negative Integer" unless size.is_a?(Integer) && size >= 0
+
       job_factory = JobFactory.new(source, options[:mutex])
       size = [job_factory.size, size].min
 
-      options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
+      discard_results = options[:discard_results]
+
+      # finish callback needs the results, careful to do that before add_progress_bar which adds finish
+      options[:discard_results] = discard_results && !options[:finish]
+
       add_progress_bar!(job_factory, options)
 
       result =
         if size == 0
+          block = ractor_block(options) if method == :in_ractors
           work_direct(job_factory, options, &block)
         elsif method == :in_threads
           work_in_threads(job_factory, options.merge(count: size), &block)
         elsif method == :in_ractors
-          work_in_ractors(job_factory, options.merge(count: size), &block)
+          work_in_ractors(job_factory, options.merge(count: size))
         else
           work_in_processes(job_factory, options.merge(count: size), &block)
         end
 
       return result.value if result.is_a?(Break)
       raise result if result.is_a?(Exception)
-      options[:return_results] ? result : source
+      discard_results ? source : result
     end
 
     def map_with_index(array, options = {}, &block)
@@ -316,7 +331,7 @@ module Parallel
     end
 
     def filter_map(...)
-      map(...).compact
+      map(...).select { |value| value }
     end
 
     # Number of physical processor cores on the current system.
@@ -377,7 +392,7 @@ module Parallel
       end
       if !result || $?.exitstatus != 0
         # Bail out if both commands returned something unexpected
-        warn "guessing pyhsical processor count"
+        warn "parallel: guessing physical processor count"
         processor_count
       else
         # powershell: "\nNumberOfCores\n-------------\n            4\n\n\n"
@@ -418,22 +433,24 @@ module Parallel
     end
 
     def work_direct(job_factory, options, &block)
+      previous_worker_number = worker_number
       self.worker_number = 0
       results = []
       exception = nil
       begin
         while (set = job_factory.next)
           item, index = set
-          results << with_instrumentation(item, index, options) do
+          result = with_instrumentation(item, index, options) do
             call_with_index(item, index, options, &block)
           end
+          results << result unless options[:discard_results]
         end
       rescue StandardError
         exception = $!
       end
       exception || results
     ensure
-      self.worker_number = nil
+      self.worker_number = previous_worker_number
     end
 
     def work_in_threads(job_factory, options, &block)
@@ -451,7 +468,7 @@ module Parallel
             result = with_instrumentation item, index, options do
               call_with_index(item, index, options, &block)
             end
-            results_mutex.synchronize { results[index] = result }
+            results_mutex.synchronize { results[index] = result } unless options[:discard_results]
           rescue StandardError
             exception = $!
           end
@@ -480,55 +497,70 @@ module Parallel
         ports[port] = ractor
       end
 
-      # start
-      ports.dup.each do |port, ractor|
-        if (job = job_factory.next)
-          item, index = job
-          instrument_start item, index, options
-          ractor.send [callback, item, index]
-        else # not enough work, `receive` would hang
-          ractor_stop ractor
-          ports.delete port
+      begin
+        # start by sending 1 item each
+        ports.dup.each do |port, ractor|
+          if (job = job_factory.next)
+            ractor_send ractor, callback, job, options
+          else # not enough work, `receive` would hang
+            ractor_stop ractor
+            ports.delete port
+          end
         end
-      end
 
-      # receive result and send new items to done ractors
-      while (job = job_factory.next)
-        # receive result
-        done_port, (exception, result, item_prev, index_prev) = Ractor.select(*ports.keys)
-        done_ractor = ports[done_port]
-        if exception
-          ports.delete done_port
-          break
-        end
-        ractor_result item_prev, index_prev, result, results, results_mutex, options
+        # receive result and send new items
+        while (job = job_factory.next)
+          # receive result
+          done_port, (exception, result, item_prev, index_prev) = Ractor.select(*ports.keys)
+          done_ractor = ports[done_port]
+          if exception
+            ractor_stop done_ractor
+            ports.delete done_port
+            break
+          end
+          ractor_result item_prev, index_prev, result, results, results_mutex, options
 
-        # send new
-        item_next, index_next = job
-        instrument_start item_next, index_next, options
-        done_ractor.send([callback, item_next, index_next])
-      end
+          # send new
+          ractor_send done_ractor, callback, job, options
+        end
 
-      # finish
-      ports.each do |port, ractor|
-        (new_exception, result, item, index) = use_port ? port.receive : ractor.take
-        exception ||= new_exception
-        next if new_exception
-        ractor_result item, index, result, results, results_mutex, options
-        ractor_stop ractor
+        # finish by receiving the last results
+        ports.dup.each do |port, ractor|
+          (new_exception, result, item, index) = use_port ? port.receive : ractor.take
+          exception ||= new_exception
+          ractor_result item, index, result, results, results_mutex, options unless new_exception
+          ractor_stop ractor
+          ports.delete port
+        end
+      ensure # close any ports that remained open in case something blew up
+        ports.each_value do |ractor|
+          ractor_stop(ractor)
+          ractor.take unless use_port
+        rescue Ractor::ClosedError, Ractor::RemoteError
+          nil
+        end
       end
 
       exception || results
     end
 
+    # ractors cannot execute blocks, so run the callback directly when not using ractors
+    def ractor_block(options)
+      klass, method_name = options[:ractor] ||
+        raise(ArgumentError, "pass the code you want to execute as `ractor: [ClassName, :method_name]`")
+      ->(*args) { klass.send(method_name, *args) }
+    end
+
     def ractor_build(use_port)
       args = use_port ? [Ractor::Port.new] : []
       ractor = Ractor.new(*args) do |port|
         loop do
-          (klass, method_name), item, index = receive
+          (klass, method_name), item, index, with_index, discard_results = receive
           break if index == :break
           begin
-            result = [nil, klass.send(method_name, item), item, index]
+            value = with_index ? klass.send(method_name, item, index) : klass.send(method_name, item)
+            value = nil if discard_results
+            result = [nil, value, item, index]
           rescue StandardError => e
             result = [e, nil, item, index]
           end
@@ -544,11 +576,19 @@ module Parallel
 
     def ractor_result(item, index, result, results, results_mutex, options)
       instrument_finish item, index, result, options
-      results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
+      results_mutex.synchronize { results[index] = result } unless options[:discard_results]
+    end
+
+    def ractor_send(ractor, callback, job, options)
+      item, index = job
+      instrument_start item, index, options
+      ractor.send [callback, item, index, options[:with_index], options[:discard_results]]
     end
 
     def ractor_stop(ractor)
       ractor.send([[nil, nil], nil, :break])
+    rescue Ractor::ClosedError
+      nil
     end
 
     def work_in_processes(job_factory, options, &blk)
@@ -579,7 +619,8 @@ module Parallel
                 result = with_instrumentation item, index, options do
                   worker.work(job_factory.pack(item, index))
                 end
-                results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
+                # arrays are not threads safe on jRuby
+                results_mutex.synchronize { results[index] = result } unless options[:discard_results]
               rescue StandardError => e
                 exception = e
                 if exception.is_a?(Kill)
@@ -617,11 +658,19 @@ module Parallel
         workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block)
       end
       workers
+    rescue Exception # rubocop:disable Lint/RescueException
+      workers.each do |worker|
+        worker.stop
+      rescue StandardError
+        nil
+      end
+      raise
     end
 
     def worker(job_factory, options, &block)
       child_read, parent_write = IO.pipe
       parent_read, child_write = IO.pipe
+      options[:serializer] ||= Serializer::Marshal
 
       pid = Process.fork do
         self.worker_number = options[:worker_number]
@@ -642,12 +691,26 @@ module Parallel
       child_read.close
       child_write.close
 
-      Worker.new(parent_read, parent_write, pid)
+      Worker.new(parent_read, parent_write, pid, options[:serializer])
+    rescue Exception # rubocop:disable Lint/RescueException
+      [child_read, parent_write, parent_read, child_write].compact.each do |io|
+        io.close unless io.closed?
+      end
+      if pid
+        UserInterruptHandler.kill(pid)
+        begin
+          Process.wait(pid)
+        rescue Errno::ECHILD
+          nil
+        end
+      end
+      raise
     end
 
     def process_incoming_jobs(read, write, job_factory, options, &block)
+      serializer = options.fetch(:serializer)
       until read.eof?
-        data = Marshal.load(read)
+        data = serializer.load(read)
         item, index = job_factory.unpack(data)
 
         result =
@@ -661,7 +724,7 @@ module Parallel
           end
 
         begin
-          Marshal.dump(result, write)
+          serializer.dump(result, write)
         rescue Errno::EPIPE
           return # parent thread already dead
         end
@@ -683,10 +746,10 @@ module Parallel
       args = [item]
       args << index if options[:with_index]
       results = block.call(*args)
-      if options[:return_results]
-        results
-      else
+      if options[:discard_results]
         nil # avoid GC overhead of passing large results around
+      else
+        results
       end
     end
 
@@ -694,7 +757,7 @@ module Parallel
       instrument_start(item, index, options)
       result = yield
       instrument_finish(item, index, result, options)
-      result unless options[:preserve_results] == false
+      result unless options[:discard_results]
     end
 
     def instrument_finish(item, index, result, options)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parallel
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.2.0
 platform: ruby
 authors:
 - Michael Grosser
@@ -16,16 +16,17 @@ extra_rdoc_files: []
 files:
 - MIT-LICENSE.txt
 - lib/parallel.rb
+- lib/parallel/serializer.rb
 - lib/parallel/version.rb
 homepage: https://github.com/grosser/parallel
 licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/grosser/parallel/issues
-  documentation_uri: https://github.com/grosser/parallel/blob/v2.0.1/Readme.md
-  source_code_uri: https://github.com/grosser/parallel/tree/v2.0.1
+  documentation_uri: https://github.com/grosser/parallel/blob/v2.2.0/Readme.md
+  source_code_uri: https://github.com/grosser/parallel/tree/v2.2.0
   wiki_uri: https://github.com/grosser/parallel/wiki
-  changelog_uri: https://github.com/grosser/parallel/blob/v2.0.1/CHANGELOG.md
+  changelog_uri: https://github.com/grosser/parallel/blob/v2.2.0/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: