parallizer 0.2.3 → 0.2.4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    parallizer (0.2.2)
+    parallizer (0.2.3)
       work_queue (~> 2.5.2)
 
 GEM
@@ -9,6 +9,7 @@ GEM
   specs:
     always_execute (0.1.3)
     diff-lcs (1.1.3)
+    rake (0.9.2.2)
     rspec (2.9.0)
       rspec-core (~> 2.9.0)
       rspec-expectations (~> 2.9.0)
@@ -25,4 +26,5 @@ PLATFORMS
 DEPENDENCIES
   always_execute (~> 0.1.1)
   parallizer!
+  rake
   rspec (~> 2.9.0)

data/lib/parallizer/thread_pool.rb

@@ -0,0 +1,63 @@
+require 'thread'
+
+class Parallizer
+  # Initially taken from Celluloid. -mgp
+  module ThreadPool
+    THEAD_POOL_SIZE = 16
+    
+    @pool = []
+    @mutex = Mutex.new
+
+    @max_idle = THEAD_POOL_SIZE
+
+    class << self
+      attr_accessor :max_idle
+
+      # Get a thread from the pool, running the given block
+      def get(&block)
+        @mutex.synchronize do
+          begin
+            if @pool.empty?
+              thread = create
+            else
+              thread = @pool.shift
+            end
+          end until thread.status # handle crashed threads
+
+          thread[:queue] << block
+          thread
+        end
+      end
+
+      # Return a thread to the pool
+      def put(thread)
+        @mutex.synchronize do
+          if @pool.size >= @max_idle
+            thread[:queue] << nil
+          else
+            @pool << thread
+          end
+        end
+      end
+
+      # Create a new thread with an associated queue of procs to run
+      def create
+        queue = Queue.new
+        thread = Thread.new do
+          while proc = queue.pop
+            begin
+              proc.call
+            rescue => ex
+              # Let this thread die on exceptions other than standard errors
+            end
+            put thread
+          end
+        end
+
+        thread[:queue] = queue
+        thread
+      end
+    end
+  end
+end
+

data/lib/parallizer/version.rb

@@ -1,3 +1,3 @@
 class Parallizer
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

data/lib/parallizer.rb

@@ -1,10 +1,10 @@
-require 'work_queue'
 require 'parallizer/version'
 require 'parallizer/proxy'
 require 'parallizer/method_call_notifier'
+require 'parallizer/thread_pool'
 
 class Parallizer
-  WORK_QUEUE_SIZE = 10
+  WORK_QUEUE_SIZE = Parallizer::ThreadPool::THEAD_POOL_SIZE
   
   attr_reader :calls, :call_infos, :client, :proxy, :options
   
@@ -49,27 +49,10 @@ class Parallizer
     Parallizer::Proxy.new(client, call_infos)
   end
   
-  def self.work_queue
-    @parallizer_work_queue ||= create_work_queue
-  end
-  
-  def self.create_work_queue
-    work_queue = WorkQueue.new(WORK_QUEUE_SIZE)
-    
-    # Force threads to be available - the work_queue gem seems to not always create the threads
-    WORK_QUEUE_SIZE.times do |i|
-      work_queue.enqueue_b do
-        1000.times { |i| i ** 1000 }
-      end
-    end
-    
-    work_queue
-  end
-  
   private
   
   def enqueue_call_info(call_info, method_name_and_args)
-    Parallizer.work_queue.enqueue_b do
+    Parallizer::ThreadPool.get do
       call_info[:mutex].synchronize do
         begin
           (call_info[:retries] + 1).times do
@@ -84,6 +67,7 @@ class Parallizer
         ensure
           call_info[:complete?] = true
           call_info[:condition_variable].broadcast
+          Parallizer::ThreadPool.put(Thread.current)
         end
       end
     end

data/parallizer.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.require_paths = ["lib"]
 
-  s.add_dependency 'work_queue', '~> 2.5.2'
+  s.add_development_dependency 'rake'
   s.add_development_dependency 'rspec', '~> 2.9.0'
   s.add_development_dependency 'always_execute', '~> 0.1.1'
 end

data/spec/parallizer_spec.rb

@@ -21,12 +21,6 @@ describe Parallizer do
     end
   end
   
-  describe "#work_queue" do
-    it "should have WORK_QUEUE_SIZE threads" do
-      Parallizer.work_queue.cur_threads.should == Parallizer::WORK_QUEUE_SIZE
-    end
-  end
-  
   describe "#add" do
     before do
       @client = TestObject.new
@@ -168,25 +162,49 @@ describe Parallizer do
     end
   end
   
-  context "with multiple threads making calls to proxy before worker executed" do
-    it "should not deadlock" do # note, this was deadlocking when using CV#signal instead of CV#broadcast
-      # force our worker thread to run after two calls from other threads
-      Parallizer::WORK_QUEUE_SIZE.times do
-        Parallizer.work_queue.enqueue_b do
-          sleep(2)
-        end
-      end
-
-      # setup the proxy
-      parallizer = Parallizer.new(TestObject.new)
-      parallizer.add.another_method
+  context "with exceptions that are not standard errors" do
+    before do
+      @retries = 3
+      @client = stub('a client')
+      @method = :a_failing_method
+      (@retries + 1).times { @client.should_receive(@method).and_raise(Exception.new('an error')) }
+    end
+    
+    execute do
+      parallizer = Parallizer.new(@client, :retries => @retries)
+      parallizer.add_call(@method)
       proxy = parallizer.create_proxy
-
-      Thread.new do
-        proxy.another_method # call in another thread must happen before call in main thread for it to deadlock
+      begin
+        proxy.send(@method)
+      rescue Exception
+        $!
       end
-      sleep(1)
-      proxy.another_method
+    end
+    
+    it "should return successful method response" do
+      @execute_result.message.should == 'an error'
     end
   end
+
+  ## Unable to repro after switch to using ThreadPool instead of work_queue gem
+  # context "with multiple threads making calls to proxy before worker executed" do
+  #   it "should not deadlock" do # note, this was deadlocking when using CV#signal instead of CV#broadcast
+  #     Parallizer::WORK_QUEUE_SIZE.times do
+  #       Parallizer::ThreadPool.get do
+  #         sleep(2)
+  #       end
+  #     end
+  # 
+  #     # setup the proxy
+  #     parallizer = Parallizer.new(TestObject.new)
+  #     parallizer.add.another_method
+  #     proxy = parallizer.create_proxy
+  # 
+  #     Thread.new do
+  #       proxy.another_method # call in another thread must happen before call in main thread for it to deadlock
+  #     end
+  #     sleep(1)
+  #     proxy.another_method
+  #   end
+  # end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parallizer
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,24 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-16 00:00:00.000000000 Z
+date: 2012-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: work_queue
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.5.2
-  type: :runtime
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.5.2
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -75,6 +75,7 @@ files:
 - lib/parallizer.rb
 - lib/parallizer/method_call_notifier.rb
 - lib/parallizer/proxy.rb
+- lib/parallizer/thread_pool.rb
 - lib/parallizer/version.rb
 - parallizer.gemspec
 - spec/parallizer/method_call_notifier_spec.rb
@@ -101,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: parallizer
-rubygems_version: 1.8.21
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Execute your service layer in parallel