parallel 1.2.4 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f6ea4deb03b39b10b4efc968f86c86d8fba2c42f
-  data.tar.gz: b32876b86f9acc449bb45d28b154708f9af0dee1
+  metadata.gz: d2df287a5ddf0d25ea8f7ab32b06e9d71046eef8
+  data.tar.gz: f4df68005a7a134922caa0dd7458a5ad48ebb14f
 SHA512:
-  metadata.gz: 92021663b44c16d0ce0f9dfa0c616f3852f9e2a6f7c85521c06283f2fd5daad75f7dbf95703241c926b71eb98a44619eefdcc62d3039e4311d8ba4389fb06649
-  data.tar.gz: c48fdaebf240f24722a41868688e0373349f78eb1de8e9329cdd0e458cdaf0458e1cbc6cbbfd4f3b5707c5363706e4755ed66c543332c03b39b2be909ccd0132
+  metadata.gz: 73b00704981e7a87f6c6448cc80759d4e4bbd62c07bc852c84c81da631480571f7acf1cff261eb72f3c2532cee96fb12670329ac9acf35c4f7c48272834ea875
+  data.tar.gz: e975280dc1eb0825479e03e2d433e89021ec336a4bc1ab3e05319afccf73d3c67f190938df1a3faeb5c9c8d1335a0ac64b67c9982c1e19936c34a83f120bdfc8

data/lib/parallel.rb

@@ -1,4 +1,3 @@
-require 'thread' # to get Thread.exclusive
 require 'rbconfig'
 require 'parallel/version'
 require 'parallel/processor_count'
@@ -15,6 +14,8 @@ module Parallel
   class Kill < StandardError
   end
 
+  Stop = Object.new
+
   INTERRUPT_SIGNAL = :SIGINT
 
   class ExceptionWrapper
@@ -47,9 +48,9 @@ module Parallel
       # process died
     end
 
-    def work(index)
+    def work(data)
       begin
-        Marshal.dump(index, write)
+        Marshal.dump(data, write)
       rescue Errno::EPIPE
         raise DeadWorker
       end
@@ -62,6 +63,66 @@ module Parallel
     end
   end
 
+  class ItemWrapper
+    def initialize(array, mutex)
+      @lambda = (array.respond_to?(:call) && array) || queue_wrapper(array)
+      @items = array.to_a unless @lambda # turn Range and other Enumerable-s into an Array
+      @mutex = mutex
+      @index = -1
+    end
+
+    def producer?
+      @lambda
+    end
+
+    def each_with_index(&block)
+      if producer?
+        loop do
+          item, index = self.next
+          break unless index
+          yield(item, index)
+        end
+      else
+        @items.each_with_index(&block)
+      end
+    end
+
+    def next
+      if producer?
+        # - index and item stay in sync
+        # - do not call lambda after it has returned Stop
+        item, index = @mutex.synchronize do
+          return if @stopped
+          item = @lambda.call
+          @stopped = (item == Parallel::Stop)
+          return if @stopped
+          [item, @index += 1]
+        end
+      else
+        index = @mutex.synchronize { @index += 1 }
+        return if index >= size
+        item = @items[index]
+      end
+      [item, index]
+    end
+
+    def size
+      @items.size
+    end
+
+    def pack(item, index)
+      producer? ? [item, index] : index
+    end
+
+    def unpack(data)
+      producer? ? data : [@items[data], data]
+    end
+
+    def queue_wrapper(array)
+      array.is_a?(::Queue) && lambda { array.pop(false) }
+    end
+  end
+
   class << self
     def in_threads(options={:count => 2})
       count, options = extract_count_from_options(options)
@@ -96,8 +157,7 @@ module Parallel
     end
 
     def map(array, options = {}, &block)
-      array = array.to_a # turn Range and other Enumerable-s into an Array
-      options[:mutex] ||= Mutex.new
+      options[:mutex] = Mutex.new
 
       if RUBY_PLATFORM =~ /java/ and not options[:in_processes]
         method = :in_threads
@@ -114,26 +174,36 @@ module Parallel
           size = 0
         end
       end
-      size = [array.size, size].min
+
+      items = ItemWrapper.new(array, options[:mutex])
+
+      size = [items.producer? ? size : items.size, size].min
 
       options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
-      add_progress_bar!(array, options)
+      add_progress_bar!(items, options)
 
       if size == 0
-        work_direct(array, options, &block)
+        work_direct(items, options, &block)
       elsif method == :in_threads
-        work_in_threads(array, options.merge(:count => size), &block)
+        work_in_threads(items, options.merge(:count => size), &block)
       else
-        work_in_processes(array, options.merge(:count => size), &block)
+        work_in_processes(items, options.merge(:count => size), &block)
       end
     end
 
-    def add_progress_bar!(array, options)
+    def map_with_index(array, options={}, &block)
+      map(array, options.merge(:with_index => true), &block)
+    end
+
+    private
+
+    def add_progress_bar!(items, options)
       if title = options[:progress]
+        raise "Progressbar and producers don't mix" if items.producer?
         require 'ruby-progressbar'
         progress = ProgressBar.create(
           :title => title,
-          :total => array.size,
+          :total => items.size,
           :format => '%t |%E | %B | %a'
         )
         old_finish = options[:finish]
@@ -144,15 +214,10 @@ module Parallel
       end
     end
 
-    def map_with_index(array, options={}, &block)
-      map(array, options.merge(:with_index => true), &block)
-    end
-
-    private
 
-    def work_direct(array, options)
+    def work_direct(items, options)
       results = []
-      array.each_with_index do |e,i|
+      items.each_with_index do |e,i|
         results << (options[:with_index] ? yield(e,i) : yield(e))
       end
       results
@@ -160,20 +225,18 @@ module Parallel
 
     def work_in_threads(items, options, &block)
       results = []
-      current = -1
       exception = nil
 
       in_threads(options[:count]) do
         # as long as there are more items, work on one of them
         loop do
           break if exception
-
-          index = Thread.exclusive { current += 1 }
-          break if index >= items.size
+          item, index = items.next
+          break unless index
 
           begin
-            results[index] = with_instrumentation items[index], index, options do
-              call_with_index(items, index, options, &block)
+            results[index] = with_instrumentation item, index, options do
+              call_with_index(item, index, options, &block)
             end
           rescue StandardError => e
             exception = e
@@ -187,9 +250,9 @@ module Parallel
 
     def work_in_processes(items, options, &blk)
       workers = create_workers(items, options, &blk)
-      current_index = -1
       results = []
       exception = nil
+
       kill_on_ctrl_c(workers.map(&:pid)) do
         in_threads(options[:count]) do |i|
           worker = workers[i]
@@ -198,11 +261,11 @@ module Parallel
           begin
             loop do
               break if exception
-              index = Thread.exclusive{ current_index += 1 }
-              break if index >= items.size
+              item, index = items.next
+              break unless index
 
-              output = with_instrumentation items[index], index, options do
-                worker.work(index)
+              output = with_instrumentation item, index, options do
+                worker.work(items.pack(item, index))
               end
 
               if ExceptionWrapper === output
@@ -264,9 +327,10 @@ module Parallel
 
     def process_incoming_jobs(read, write, items, options, &block)
       while !read.eof?
-        index = Marshal.load(read)
+        data = Marshal.load(read)
+        item, index = items.unpack(data)
         result = begin
-          call_with_index(items, index, options, &block)
+          call_with_index(item, index, options, &block)
         rescue StandardError => e
           ExceptionWrapper.new(e)
         end
@@ -355,8 +419,8 @@ module Parallel
       end
     end
 
-    def call_with_index(array, index, options, &block)
-      args = [array[index]]
+    def call_with_index(item, index, options, &block)
+      args = [item]
       args << index if options[:with_index]
       if options[:return_results]
         block.call(*args)

data/lib/parallel/version.rb

@@ -1,3 +1,3 @@
 module Parallel
-  VERSION = Version = '1.2.4'
+  VERSION = Version = '1.3.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parallel
 version: !ruby/object:Gem::Version
-  version: 1.2.4
+  version: 1.3.0
 platform: ruby
 authors:
 - Michael Grosser
@@ -30,7 +30,7 @@ cert_chain:
   F5etKHZg0j3eHO31/i2HnswY04lqGImUu6aM5EnijFTB7PPW2KwKKM4+kKDYFdlw
   /0WV1Ng2/Y6qsHwmqGg2VlYj2h4=
   -----END CERTIFICATE-----
-date: 2014-08-17 00:00:00.000000000 Z
+date: 2014-08-30 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: michael@grosser.it