in_threads 1.0.0 → 1.1.0

data/in_threads.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'in_threads'
-  s.version     = '1.0.0'
+  s.version     = '1.1.0'
   s.summary     = %q{Execute ruby code in parallel}
   s.homepage    = "http://github.com/toy/#{s.name}"
   s.authors     = ['Ivan Kuchin']

data/lib/in_threads.rb

@@ -3,63 +3,112 @@ require 'thwait'
 
 module Enumerable
   # Run enumerable method blocks in threads
+  #
+  #   urls.in_threads.map do |url|
+  #     url.fetch
+  #   end
+  #
+  # Specify number of threads to use:
+  #
+  #   files.in_threads(4).all? do |file|
+  #     file.valid?
+  #   end
+  #
+  # Passing block runs it against <tt>each</tt>
+  #
+  #   urls.in_threads.each{ … }
+  #
+  # is same as
+  #
+  #   urls.in_threads{ … }
   def in_threads(thread_count = 10, &block)
     InThreads.new(self, thread_count, &block)
   end
 end
 
-# TODO: create my own ThreadsWait with blackjack and hookers?
-# TODO: create class methods for connecting Enumerable method to runner
-# TODO: run_in_threads_inconsecutive for `all?`, `any?`, `none?` and `one?`
 # TODO: all ruby1.9.3 methods
-# TODO: better way of handling grep?
-# TODO: check method presence if Enumerable before connectin to runner
 
 class InThreads
   (
     instance_methods.map(&:to_s) -
     %w[__id__ __send__ class inspect instance_of? is_a? kind_of? nil? object_id respond_to? send]
   ).each{ |name| undef_method name }
-  (private_instance_methods.map(&:to_s) - %w[initialize]).each{ |name| undef_method name }
+  (private_instance_methods.map(&:to_s) - %w[initialize raise]).each{ |name| undef_method name }
 
   attr_reader :enumerable, :thread_count
   def initialize(enumerable, thread_count = 10, &block)
-    @enumerable, @thread_count = enumerable, thread_count
+    @enumerable, @thread_count = enumerable, thread_count.to_i
+    unless enumerable.class.include?(Enumerable)
+      raise ArgumentError.new('`enumerable` should include Enumerable.')
+    end
+    if thread_count < 2
+      raise ArgumentError.new('`thread_count` can\'t be less than 2.')
+    end
     each(&block) if block
   end
 
+  # Creates new instance using underlying enumerable and new thread_count
   def in_threads(thread_count = 10, &block)
     self.class.new(enumerable, thread_count, &block)
   end
 
-  %w[
-    each
+  class << self
+    # List of instance_methods of Enumerable
+    def enumerable_methods
+      Enumerable.instance_methods.map(&:to_s)
+    end
+
+    # Specify runner to use
+    #
+    #   use :run_in_threads_consecutive, :for => %w[all? any? none? one?]
+    #
+    # <tt>:for</tt> is required
+    # <tt>:ignore_undefined</tt> ignores methods which are not present in list returned by <tt>enumerable_methods</tt>
+    def use(runner, options)
+      methods = Array(options[:for])
+      raise 'no methods provided using :for option' if methods.empty?
+      ignore_undefined = options[:ignore_undefined]
+      enumerable_methods = self.enumerable_methods
+      methods.each do |method|
+        unless ignore_undefined && !enumerable_methods.include?(method)
+          class_eval <<-RUBY
+            def #{method}(*args, &block)
+              #{runner}(enumerable, :#{method}, *args, &block)
+            end
+          RUBY
+        end
+      end
+    end
+  end
+
+  use :run_in_threads_block_result_irrelevant, :for => %w[each]
+  use :run_in_threads_consecutive, :for => %w[
     all? any? none? one?
     detect find find_index drop_while take_while
     partition find_all select reject count
     collect map group_by max_by min_by minmax_by sort_by
-  ].each do |name|
-    class_eval <<-RUBY
-      def #{name}(*args, &block)
-        run_in_threads_consecutive(enumerable, :#{name}, *args, &block)
-      end
-    RUBY
-  end
-
-  %w[
+    flat_map collect_concat
+  ], :ignore_undefined => true
+  use :run_in_threads_block_result_irrelevant, :for => %w[
     reverse_each
     each_with_index enum_with_index
     each_cons each_slice enum_cons enum_slice
     zip
     cycle
-  ].each do |name|
-    class_eval <<-RUBY
-      def #{name}(*args, &block)
-        run_in_threads_block_result_irrelevant(enumerable, :#{name}, *args, &block)
-      end
-    RUBY
-  end
+    each_entry
+  ], :ignore_undefined => true
+  use :run_without_threads, :for => %w[
+    inject reduce
+    max min minmax sort
+    entries to_a
+    drop take
+    first
+    include? member?
+    each_with_object
+    chunk slice_before
+  ], :ignore_undefined => true
 
+  # Special case method, works by applying <tt>run_in_threads_consecutive</tt> with map on enumerable returned by blockless run
   def grep(*args, &block)
     if block
       run_in_threads_consecutive(enumerable.grep(*args), :map, &block)
@@ -68,63 +117,75 @@ class InThreads
     end
   end
 
-  %w[
-    inject reduce
-    max min minmax sort
-    entries to_a
-    drop take
-    first
-    include? member?
-  ].each do |name|
-    class_eval <<-RUBY
-      def #{name}(*args, &block)
-        enumerable.#{name}(*args, &block)
+protected
+
+  # Use ThreadsWait to limit number of threads
+  class ThreadLimiter
+    # Initialize with limit
+    def initialize(count)
+      @count = count
+      @waiter = ThreadsWait.new
+    end
+
+    # Without block behaves as <tt>new</tt>
+    # With block yields it with <tt>self</tt> and ensures running of <tt>finalize</tt>
+    def self.limit(count, &block)
+      limiter = new(count)
+      if block
+        begin
+          yield limiter
+        ensure
+          limiter.finalize
+        end
+      else
+        limiter
       end
-    RUBY
-  end
+    end
 
-private
+    # Add thread to <tt>ThreadsWait</tt>, wait for finishing of one thread if limit reached
+    def add(thread)
+      if @waiter.threads.length + 1 >= @count
+        @waiter.join(thread)
+      else
+        @waiter.join_nowait(thread)
+      end
+    end
+
+    # Wait for waiting threads
+    def finalize
+      @waiter.all_waits
+    end
+  end
 
+  # Use for methods which don't use block result
   def run_in_threads_block_result_irrelevant(enumerable, method, *args, &block)
     if block
-      waiter = ThreadsWait.new
-      begin
+      ThreadLimiter.limit(thread_count) do |limiter|
         enumerable.send(method, *args) do |*block_args|
-          waiter.next_wait if waiter.threads.length >= thread_count
-          waiter.join_nowait([Thread.new(*block_args, &block)])
+          limiter.add(Thread.new(*block_args, &block))
         end
-      ensure
-        waiter.all_waits
       end
     else
       enumerable.send(method, *args)
     end
   end
 
+  # Use for methods which do use block result and fire objects in same way as <tt>each</tt>
   def run_in_threads_consecutive(enumerable, method, *args, &block)
     if block
       begin
         queue = Queue.new
-        runner = Thread.new(enumerable) do |enumerable|
-          threads = []
-          begin
+        runner = Thread.new do
+          ThreadLimiter.limit(thread_count) do |limiter|
             enumerable.each do |object|
-              if threads.length >= thread_count
-                threads = threads.select(&:alive?)
-                if threads.length >= thread_count
-                  ThreadsWait.new(*threads).next_wait
-                end
-              end
               break if Thread.current[:stop]
               thread = Thread.new(object, &block)
-              threads << thread
               queue << thread
+              limiter.add(thread)
             end
-          ensure
-            threads.map(&:join)
           end
         end
-        enumerable.send(method, *args) do |*block_args|
+        enumerable.send(method, *args) do |object|
           queue.pop.value
         end
       ensure
@@ -135,4 +196,9 @@ private
       enumerable.send(method, *args)
     end
   end
+
+  # Use for methods which don't use blocks or can not use threads
+  def run_without_threads(enumerable, method, *args, &block)
+    enumerable.send(method, *args, &block)
+  end
 end

data/spec/in_threads_spec.rb

@@ -48,8 +48,16 @@ class ValueItem < Item
   end
 end
 
-def enum_methods(methods)
-  (Enumerable.instance_methods.map(&:to_s) & methods)
+def describe_enum_method(method, &block)
+  @enum_methods ||= Enumerable.instance_methods.map(&:to_s)
+  if @enum_methods.include?(method)
+    describe(method, &block)
+  else
+    it "should not be defined" do
+      exception_regexp = /^undefined method `#{Regexp.escape(method)}' for #<InThreads:0x[0-9a-f]+>$/
+      proc{ enum.in_threads.send(method) }.should raise_error(NoMethodError, exception_regexp)
+    end
+  end
 end
 
 describe "in_threads" do
@@ -62,6 +70,30 @@ describe "in_threads" do
     Time.now - start
   end
 
+  (Enumerable.instance_methods - 10.times.in_threads.class.instance_methods).each do |method|
+    pending method
+  end
+
+  describe "verifying params" do
+    it "should complain about using with non enumerable" do
+      proc{ InThreads.new(1) }.should raise_error(ArgumentError)
+    end
+
+    [1..10, 10.times, {}, []].each do |o|
+      it "should complain about using with #{o.class}" do
+        proc{ InThreads.new(o) }.should_not raise_error
+      end
+    end
+
+    it "should complain about using less than 2 threads" do
+      proc{ 10.times.in_threads(1) }.should raise_error(ArgumentError)
+    end
+
+    it "should not complain about using 2 or more threads" do
+      proc{ 10.times.in_threads(2) }.should_not raise_error
+    end
+  end
+
   describe "in_threads" do
     it "should not change existing instance" do
       threaded = enum.in_threads(10)
@@ -79,6 +111,31 @@ describe "in_threads" do
     end
   end
 
+  describe "thread count" do
+    let(:enum){ 100.times.map{ |i| ValueItem.new(i, i < 50) } }
+
+    %w[each map all?].each do |method|
+      it "should run in specified number of threads for #{method}" do
+        @thread_count = 0
+        @max_thread_count = 0
+        @mutex = Mutex.new
+        enum.in_threads(13).send(method) do |o|
+          @mutex.synchronize do
+            @thread_count += 1
+            @max_thread_count = [@max_thread_count, @thread_count].max
+          end
+          res = o.check?
+          @mutex.synchronize do
+            @thread_count -= 1
+          end
+          res
+        end
+        @thread_count.should == 0
+        @max_thread_count.should == 13
+      end
+    end
+  end
+
   describe "each" do
     it "should return same enum after running" do
       enum.in_threads.each(&:value).should == enum
@@ -86,7 +143,7 @@ describe "in_threads" do
 
     it "should execute block for each element" do
       enum.each{ |o| o.should_receive(:touch).once }
-      enum.in_threads.each(&:touch)
+      enum.in_threads.each(&:touch_n_value)
     end
 
     it "should run faster with threads" do
@@ -94,7 +151,7 @@ describe "in_threads" do
     end
 
     it "should run faster with more threads" do
-      measure{ enum.in_threads(20).each(&:work) }.should < measure{ enum.in_threads(2).each(&:work) } * speed_coef
+      measure{ enum.in_threads(10).each(&:work) }.should < measure{ enum.in_threads(2).each(&:work) } * speed_coef
     end
 
     it "should return same enum without block" do
@@ -102,8 +159,8 @@ describe "in_threads" do
     end
   end
 
-  enum_methods(%w[each_with_index enum_with_index]).each do |method|
-    describe method do
+  %w[each_with_index enum_with_index].each do |method|
+    describe_enum_method method do
       let(:runner){ proc{ |o, i| o.value } }
 
       it "should return same result with threads" do
@@ -131,14 +188,14 @@ describe "in_threads" do
     end
 
     it "should fire same objects in reverse order" do
-      @order = mock('order', :touch => nil)
-      @order.should_receive(:touch).with(enum.last).ordered
-      @order.should_receive(:touch).with(enum[enum.length / 2]).ordered
-      @order.should_receive(:touch).with(enum.first).ordered
+      @order = mock('order', :notify => nil)
+      @order.should_receive(:notify).with(enum.last).ordered
+      @order.should_receive(:notify).with(enum[enum.length / 2]).ordered
+      @order.should_receive(:notify).with(enum.first).ordered
       enum.reverse_each{ |o| o.should_receive(:touch).once }
       @mutex = Mutex.new
       enum.in_threads.reverse_each do |o|
-        @mutex.synchronize{ @order.touch(o) }
+        @mutex.synchronize{ @order.notify(o) }
         o.touch_n_value
       end
     end
@@ -220,8 +277,8 @@ describe "in_threads" do
     end
   end
 
-  enum_methods(%w[each_cons each_slice enum_slice enum_cons]).each do |method|
-    describe method do
+  %w[each_cons each_slice enum_slice enum_cons].each do |method|
+    describe_enum_method method do
       let(:runner){ proc{ |a| a.each(&:value) } }
 
       it "should fire same objects" do
@@ -300,6 +357,68 @@ describe "in_threads" do
     end
   end
 
+  describe_enum_method "each_entry" do
+    class EachEntryYielder
+      include Enumerable
+      def each
+        10.times{ yield 1 }
+        10.times{ yield 2, 3 }
+        10.times{ yield }
+      end
+    end
+
+    let(:enum){ EachEntryYielder.new }
+    let(:runner){ proc{ |o| ValueItem.new(0, o).work } }
+
+    it "should return same result with threads" do
+      enum.in_threads.each_entry(&runner).should == enum.each_entry(&runner)
+    end
+
+    it "should execute block for each element" do
+      @order = mock('order')
+      @order.should_receive(:notify).with(1).exactly(10).times.ordered
+      @order.should_receive(:notify).with([2, 3]).exactly(10).times.ordered
+      @order.should_receive(:notify).with(nil).exactly(10).times.ordered
+      @mutex = Mutex.new
+      enum.in_threads.each_entry do |o|
+        @mutex.synchronize{ @order.notify(o) }
+        runner[]
+      end
+    end
+
+    it "should run faster with threads" do
+      measure{ enum.in_threads.each_entry(&runner) }.should < measure{ enum.each_entry(&runner) } * speed_coef
+    end
+
+    it "should return same enum without block" do
+      enum.in_threads.each_entry.to_a.should == enum.each_entry.to_a
+    end
+  end
+
+  %w[flat_map collect_concat].each do |method|
+    describe_enum_method method do
+      let(:enum){ 20.times.map{ |i| Item.new(i) }.each_slice(3) }
+      let(:runner){ proc{ |a| a.map(&:value) } }
+
+      it "should return same result with threads" do
+        enum.in_threads.send(method, &runner).should == enum.send(method, &runner)
+      end
+
+      it "should fire same objects" do
+        enum.send(method){ |a| a.each{ |o| o.should_receive(:touch).with(a).once } }
+        enum.in_threads.send(method){ |a| a.each{ |o| o.touch_n_value(a) } }
+      end
+
+      it "should run faster with threads" do
+        measure{ enum.in_threads.send(method, &runner) }.should < measure{ enum.send(method, &runner) } * speed_coef
+      end
+
+      it "should return same enum without block" do
+        enum.in_threads.send(method).to_a.should == enum.send(method).to_a
+      end
+    end
+  end
+
   context "unthreaded" do
     %w[inject reduce].each do |method|
       describe method do
@@ -351,5 +470,21 @@ describe "in_threads" do
         end
       end
     end
+
+    describe_enum_method "each_with_object" do
+      let(:runner){ proc{ |o, h| h[o.value] = true } }
+
+      it "should return same result" do
+        enum.in_threads.each_with_object({}, &runner).should == enum.each_with_object({}, &runner)
+      end
+    end
+
+    %w[chunk slice_before].each do |method|
+      describe_enum_method method do
+        it "should return same result" do
+          enum.in_threads.send(method, &:check?).to_a.should == enum.send(method, &:check?).to_a
+        end
+      end
+    end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: in_threads
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 19
   prerelease: 
   segments: 
   - 1
+  - 1
   - 0
-  - 0
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors: 
 - Ivan Kuchin
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-12-05 00:00:00 Z
+date: 2011-12-08 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec