parallel 1.4.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca1dbd921b5d79d0ec3ece5ad278ba4e5f06416f
-  data.tar.gz: aba073a0c18c20be741892f32ba0ff7c2421a50a
+  metadata.gz: 4ebdef905e948776369686271708c85a60b4c63c
+  data.tar.gz: da64471e0d337ca3a778670efed6e0c2eda454af
 SHA512:
-  metadata.gz: 8ed2cc642c77c45eb234d18f9a39a53891d16a3f55cde102a91167e8e1b0a0fe1120f9e33da692b08fabd676265d8c471fc6668f64b714fc6f4d93410cea4723
-  data.tar.gz: 74b5c077ffd1ae61e3bf4e1b8cf8f5662058b2bbe73adf8882f212a1c72a183545767eb86e2a7647ca1dbba422beabab5b76d5822dc1f3537a77be1f46ff4bb6
+  metadata.gz: 9677d167fc5b5796630d9654bc5dd82b1e950aa0a69537dcdb1332326ac95399d91638827cb69a00be79123db43514586d30037a92a9572dba84ff63745b4ff1
+  data.tar.gz: 4dca43cfa0a92caf06ca45b95de2a3b28e781b6e0300b94c14fb4703a1f4a264d73c10c2d11b2e92c344e92c998afeb60245bd1aafb100c3e7d262ca88fd665f

data/lib/parallel.rb

@@ -16,8 +16,6 @@ module Parallel
 
   Stop = Object.new
 
-  INTERRUPT_SIGNAL = :SIGINT
-
   class ExceptionWrapper
     attr_reader :exception
     def initialize(exception)
@@ -65,31 +63,15 @@ module Parallel
     end
   end
 
-  class ItemWrapper
-    def initialize(array, mutex)
-      @lambda = (array.respond_to?(:call) && array) || queue_wrapper(array)
-      @items = array.to_a unless @lambda # turn Range and other Enumerable-s into an Array
+  class JobFactory
+    def initialize(source, mutex)
+      @lambda = (source.respond_to?(:call) && source) || queue_wrapper(source)
+      @source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array
       @mutex = mutex
       @index = -1
       @stopped = false
     end
 
-    def producer?
-      @lambda
-    end
-
-    def each_with_index(&block)
-      if producer?
-        loop do
-          item, index = self.next
-          break unless index
-          yield(item, index)
-        end
-      else
-        @items.each_with_index(&block)
-      end
-    end
-
     def next
       if producer?
         # - index and item stay in sync
@@ -104,13 +86,17 @@ module Parallel
       else
         index = @mutex.synchronize { @index += 1 }
         return if index >= size
-        item = @items[index]
+        item = @source[index]
       end
       [item, index]
     end
 
     def size
-      @items.size
+      if producer?
+        Float::INFINITY
+      else
+        @source.size
+      end
     end
 
     def pack(item, index)
@@ -118,7 +104,13 @@ module Parallel
     end
 
     def unpack(data)
-      producer? ? data : [@items[data], data]
+      producer? ? data : [@source[data], data]
+    end
+
+    private
+
+    def producer?
+      @lambda
     end
 
     def queue_wrapper(array)
@@ -126,6 +118,68 @@ module Parallel
     end
   end
 
+  class UserInterruptHandler
+    INTERRUPT_SIGNAL = :SIGINT
+
+    class << self
+      # kill all these pids or threads if user presses Ctrl+c
+      def kill_on_ctrl_c(things, options)
+        return yield if RUBY_ENGINE == "jruby"
+        @to_be_killed ||= []
+        old_interrupt = nil
+        signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
+
+        if @to_be_killed.empty?
+          old_interrupt = trap_interrupt(signal) do
+            $stderr.puts 'Parallel execution interrupted, exiting ...'
+            @to_be_killed.flatten.compact.each { |thing| kill(thing) }
+          end
+        end
+
+        @to_be_killed << things
+
+        yield
+      ensure
+        @to_be_killed.pop # free threads for GC and do not kill pids that could be used for new processes
+        restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
+      end
+
+      def kill(thing)
+        if thing.is_a?(Thread)
+          thing.kill
+        else
+          begin
+            Process.kill(:KILL, thing)
+          rescue Errno::ESRCH
+            # some linux systems already automatically killed the children at this point
+            # so we just ignore them not being there
+          end
+        end
+      end
+
+      private
+
+      def trap_interrupt(signal)
+        old = Signal.trap signal, 'IGNORE'
+
+        Signal.trap signal do
+          yield
+          if old == "DEFAULT"
+            raise Interrupt
+          else
+            old.call
+          end
+        end
+
+        old
+      end
+
+      def restore_interrupt(old, signal)
+        Signal.trap signal, old
+      end
+    end
+  end
+
   class << self
     def in_threads(options={:count => 2})
       count, options = extract_count_from_options(options)
@@ -139,7 +193,7 @@ module Parallel
         end
       end
 
-      kill_on_ctrl_c(threads, options) { wait_for_threads(threads) }
+      UserInterruptHandler.kill_on_ctrl_c(threads, options) { wait_for_threads(threads) }
 
       out
     end
@@ -159,7 +213,7 @@ module Parallel
       each(array, options.merge(:with_index => true), &block)
     end
 
-    def map(array, options = {}, &block)
+    def map(source, options = {}, &block)
       options[:mutex] = Mutex.new
 
       if RUBY_PLATFORM =~ /java/ and not options[:in_processes]
@@ -173,24 +227,23 @@ module Parallel
         if Process.respond_to?(:fork)
           size = options[method] || processor_count
         else
-          $stderr.puts "Warning: Process.fork is not supported by this Ruby"
+          warn "Process.fork is not supported by this Ruby"
           size = 0
         end
       end
 
-      items = ItemWrapper.new(array, options[:mutex])
-
-      size = [items.producer? ? size : items.size, size].min
+      job_factory = JobFactory.new(source, options[:mutex])
+      size = [job_factory.size, size].min
 
       options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
-      add_progress_bar!(items, options)
+      add_progress_bar!(job_factory, options)
 
       if size == 0
-        work_direct(items, options, &block)
+        work_direct(job_factory, options, &block)
       elsif method == :in_threads
-        work_in_threads(items, options.merge(:count => size), &block)
+        work_in_threads(job_factory, options.merge(:count => size), &block)
       else
-        work_in_processes(items, options.merge(:count => size), &block)
+        work_in_processes(job_factory, options.merge(:count => size), &block)
       end
     end
 
@@ -200,9 +253,9 @@ module Parallel
 
     private
 
-    def add_progress_bar!(items, options)
+    def add_progress_bar!(job_factory, options)
       if progress_options = options[:progress]
-        raise "Progressbar and producers don't mix" if items.producer?
+        raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
         require 'ruby-progressbar'
 
         if progress_options.respond_to? :to_str
@@ -210,7 +263,7 @@ module Parallel
         end
 
         progress_options = {
-          total: items.size,
+          total: job_factory.size,
           format: '%t |%E | %B | %a'
         }.merge(progress_options)
 
@@ -223,10 +276,10 @@ module Parallel
       end
     end
 
-
-    def work_direct(items, options, &block)
+    def work_direct(job_factory, options, &block)
       results = []
-      items.each_with_index do |item, index|
+      while set = job_factory.next
+        item, index = set
         results << with_instrumentation(item, index, options) do
           call_with_index(item, index, options, &block)
         end
@@ -234,24 +287,20 @@ module Parallel
       results
     end
 
-    def work_in_threads(items, options, &block)
+    def work_in_threads(job_factory, options, &block)
       results = []
       exception = nil
 
       in_threads(options) do
-        # as long as there are more items, work on one of them
-        loop do
-          break if exception
-          item, index = items.next
-          break unless index
-
+        # as long as there are more jobs, work on one of them
+        while !exception && set = job_factory.next
           begin
+            item, index = set
             results[index] = with_instrumentation item, index, options do
               call_with_index(item, index, options, &block)
             end
           rescue StandardError => e
             exception = e
-            break
           end
         end
       end
@@ -259,12 +308,12 @@ module Parallel
       handle_exception(exception, results)
     end
 
-    def work_in_processes(items, options, &blk)
-      workers = create_workers(items, options, &blk)
+    def work_in_processes(job_factory, options, &blk)
+      workers = create_workers(job_factory, options, &blk)
       results = []
       exception = nil
 
-      kill_on_ctrl_c(workers.map(&:pid), options) do
+      UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
         in_threads(options) do |i|
           worker = workers[i]
           worker.thread = Thread.current
@@ -272,19 +321,19 @@ module Parallel
           begin
             loop do
               break if exception
-              item, index = items.next
+              item, index = job_factory.next
               break unless index
 
               begin
                 results[index] = with_instrumentation item, index, options do
-                  worker.work(items.pack(item, index))
+                  worker.work(job_factory.pack(item, index))
                 end
               rescue StandardError => e
                 exception = e
                 if Parallel::Kill === exception
                   (workers - [worker]).each do |w|
-                    kill_that_thing!(w.thread)
-                    kill_that_thing!(w.pid)
+                    UserInterruptHandler.kill(w.thread)
+                    UserInterruptHandler.kill(w.pid)
                   end
                 end
               end
@@ -299,18 +348,15 @@ module Parallel
       handle_exception(exception, results)
     end
 
-    def create_workers(items, options, &block)
+    def create_workers(job_factory, options, &block)
       workers = []
       Array.new(options[:count]).each do
-        workers << worker(items, options.merge(:started_workers => workers), &block)
+        workers << worker(job_factory, options.merge(:started_workers => workers), &block)
       end
       workers
     end
 
-    def worker(items, options, &block)
-      # use less memory on REE
-      GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
-
+    def worker(job_factory, options, &block)
       child_read, parent_write = IO.pipe
       parent_read, child_write = IO.pipe
 
@@ -321,7 +367,7 @@ module Parallel
           parent_write.close
           parent_read.close
 
-          process_incoming_jobs(child_read, child_write, items, options, &block)
+          process_incoming_jobs(child_read, child_write, job_factory, options, &block)
         ensure
           child_read.close
           child_write.close
@@ -334,10 +380,10 @@ module Parallel
       Worker.new(parent_read, parent_write, pid)
     end
 
-    def process_incoming_jobs(read, write, items, options, &block)
-      while !read.eof?
+    def process_incoming_jobs(read, write, job_factory, options, &block)
+      until read.eof?
         data = Marshal.load(read)
-        item, index = items.unpack(data)
+        item, index = job_factory.unpack(data)
         result = begin
           call_with_index(item, index, options, &block)
         rescue StandardError => e
@@ -376,59 +422,6 @@ module Parallel
       [count, options]
     end
 
-    # kill all these pids or threads if user presses Ctrl+c
-    def kill_on_ctrl_c(things, options)
-      @to_be_killed ||= []
-      old_interrupt = nil
-      signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
-
-      if @to_be_killed.empty?
-        old_interrupt = trap_interrupt(signal) do
-          $stderr.puts 'Parallel execution interrupted, exiting ...'
-          @to_be_killed.flatten.compact.each { |thing| kill_that_thing!(thing) }
-        end
-      end
-
-      @to_be_killed << things
-
-      yield
-    ensure
-      @to_be_killed.pop # free threads for GC and do not kill pids that could be used for new processes
-      restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
-    end
-
-    def trap_interrupt(signal)
-      old = Signal.trap signal, 'IGNORE'
-
-      Signal.trap signal do
-        yield
-        if old == "DEFAULT"
-          raise Interrupt
-        else
-          old.call
-        end
-      end
-
-      old
-    end
-
-    def restore_interrupt(old, signal)
-      Signal.trap signal, old
-    end
-
-    def kill_that_thing!(thing)
-      if thing.is_a?(Thread)
-        thing.kill
-      else
-        begin
-          Process.kill(:KILL, thing)
-        rescue Errno::ESRCH
-          # some linux systems already automatically killed the children at this point
-          # so we just ignore them not being there
-        end
-      end
-    end
-
     def call_with_index(item, index, options, &block)
       args = [item]
       args << index if options[:with_index]

data/lib/parallel/version.rb

@@ -1,3 +1,3 @@
 module Parallel
-  VERSION = Version = '1.4.1'
+  VERSION = Version = '1.5.0'
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: parallel
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Michael Grosser
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-01 00:00:00.000000000 Z
+date: 2015-05-25 00:00:00.000000000 Z
 dependencies: []
-description: 
+description:
 email: michael@grosser.it
 executables: []
 extensions: []
@@ -24,24 +24,24 @@ homepage: https://github.com/grosser/parallel
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
 specification_version: 4
 summary: Run any kind of code in parallel processes
 test_files: []