chantier 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c4e130a470bcbd6888307a6130b049ada83ec34
-  data.tar.gz: aac008cb10feec2ab5e804a5c6bccfb77c660bbf
+  metadata.gz: 748fbefb75937019db2ddbf9d8ff6ffcd88c1227
+  data.tar.gz: cacf4320b37f1891393ec55a8ed43d316825d21b
 SHA512:
-  metadata.gz: 3daeffddbc5134d1247c2af7edeb109011f25072f4428d10805306c005f6236c74a1779dcf66993d8e741f5cfb1544d6b44ffae8e74b7944a52a324afa2f87e8
-  data.tar.gz: d58a273d22173f4de4c07f49cf1102d0ee4b91c2c31a72b29ddf063f7a263a4b20fd6e5f361c425b0598ca92afe721fc25173d345674af5cb59bc2316444e436
+  metadata.gz: 0c14a5d0828f0ed0f4491abdb6e152f027e73d457869f90e4ec3bf9630deb819f0ffd3e710814bfa94b654dc4a25403efdf1fe27800aa4b164af123d0852ff42
+  data.tar.gz: b6382976248c36e1fa908dcd3f4b80162f4534c5392de7ceecf0687446d8a5aa2b727507f3bdf6ccff466f14ff9818aa36a25ba26bb6b455dfe72befd57d9369

data/chantier.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: chantier 0.0.4 ruby lib
+# stub: chantier 0.0.5 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "chantier"
-  s.version = "0.0.4"
+  s.version = "0.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2014-08-03"
+  s.date = "2014-08-05"
   s.description = " Process your jobs in parallel with a simple table of processes or threads "
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [

data/lib/chantier.rb

@@ -1,5 +1,5 @@
 module Chantier
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
   require_relative 'process_pool'
   require_relative 'process_pool_with_kill'
   require_relative 'thread_pool'

data/lib/process_pool.rb

@@ -34,7 +34,13 @@ class Chantier::ProcessPool
   # the loop returns.
   SCHEDULER_SLEEP_SECONDS = (1.0 / 1000)
   
-  def initialize(num_procs)
+  # Initializes a new ProcessPool with the given number of workers. If max_failures is
+  # given the fork_task method will raise an exception if more than N processes spawned
+  # have been terminated with a non-0 exit status.
+  def initialize(num_procs, max_failures: nil)
+    @max_failures = max_failures && max_failures.to_i
+    @non_zero_exits = 0
+    
     raise "Need at least 1 slot, given #{num_procs.to_i}" unless num_procs.to_i > 0
     @pids = [nil] * num_procs.to_i
     @semaphore = Mutex.new
@@ -62,6 +68,10 @@ class Chantier::ProcessPool
   # becomes free. Once that happens, the given block will be forked off
   # and the method will return.
   def fork_task(&blk)
+    if @max_failures && @non_zero_exits > @max_failures
+      raise "Reached error limit of processes quitting with non-0 status - limit set at #{@max_failures}"
+    end
+    
     destination_slot_idx = nil
     
     # Try to find a slot in the process table where this job can go
@@ -88,8 +98,13 @@ class Chantier::ProcessPool
     # process table
     Thread.new do
       Process.wait(task_pid) # This call will block until that process quites
-      # Now we can remove that process from the process table
-      @semaphore.synchronize { @pids[destination_slot_idx] = nil }
+      terminated_normally = $?.exited? && $?.exitstatus.zero?
+      @semaphore.synchronize do
+        # Now we can remove that process from the process table
+        @pids[destination_slot_idx] = nil
+        # and increment the error count if needed
+        @non_zero_exits += 1 unless terminated_normally
+      end
     end
     
     return task_pid

data/lib/thread_pool.rb

@@ -34,10 +34,21 @@ class Chantier::ThreadPool
   # the loop returns.
   SCHEDULER_SLEEP_SECONDS = (1.0 / 500)
   
-  def initialize(num_threads)
+  # Initializes a new ProcessPool with the given number of workers. If max_failures is
+  # given the fork_task method will raise an exception if more than N threads spawned
+  # have raised during execution.
+  def initialize(num_threads, max_failures: nil)
     raise "Need at least 1 slot, given #{num_threads.to_i}" unless num_threads.to_i > 0
     @threads = [nil] * num_threads.to_i
     @semaphore = Mutex.new
+    
+    # Failure counters
+    @failure_count = 0
+    @max_failures = max_failures && max_failures.to_i
+    
+    # Information on the last exception that happened
+    @aborted = false
+    @last_representative_exception = nil
   end
   
   # Distributes the elements in the given Enumerable to parallel workers,
@@ -61,6 +72,10 @@ class Chantier::ThreadPool
   # the thread it is called from until a slot in the thread table
   # becomes free.
   def fork_task(&blk)
+    if @last_representative_exception
+      raise "Reached error limit of #{@max_failures} (last error was #{@last_representative_exception.inspect})"
+    end
+    
     destination_slot_idx = nil
     
     # Try to find a slot in the process table where this job can go
@@ -78,11 +93,11 @@ class Chantier::ThreadPool
     
     # No need to lock this because we already reserved that slot
     @threads[destination_slot_idx] = Thread.new do
-      yield
-      # Now we can remove that process from the process table
+      # Run the given block
+      run_block_with_exception_protection(&blk)
+      # ...and remove that process from the process table
       @semaphore.synchronize { @threads[destination_slot_idx] = nil }
     end
-    
   end
   
   # Tells whether some processes are still churning
@@ -98,4 +113,20 @@ class Chantier::ThreadPool
       end
     end
   end
+  
+  private
+  
+  def run_block_with_exception_protection(&blk)
+    yield
+  rescue Exception => e
+    # Register the failure and decrement the counter. If we had more than N 
+    # failures stop the machine completely by raising an exception in the caller.
+    @semaphore.synchronize do
+      @failure_count += 1
+      if @max_failures && (@failure_count > @max_failures)
+        @last_representative_exception = e
+      end
+    end
+  end
+  
 end

data/spec/process_pool_spec.rb

@@ -50,6 +50,27 @@ describe Chantier::ProcessPool do
     Chantier::ProcessPool.new(10)
   end
   
+  context 'with failures' do
+    it 'raises after 4 failures' do
+      under_test = described_class.new(num_workers=3, max_failures: '4')
+      expect {
+        15.times do 
+          under_test.fork_task { raise "I am such a failure" }
+        end
+      }.to raise_error('Reached error limit of processes quitting with non-0 status - limit set at 4')
+    end
+    
+    it 'runs through the jobs if max_failures is not given' do
+      under_test = described_class.new(num_workers=3)
+      7.times {
+        under_test.fork_task { raise "I am such a failure" }
+      }
+      under_test.block_until_complete!
+      expect(true).to eq(true), "Should have gotten to this assertion without the Pool blocking"
+    end
+  end
+  
+  
   context 'with 1 slot' do
     let(:manager) { described_class.new(1) }
     

data/spec/thread_pool_spec.rb

@@ -46,6 +46,26 @@ describe Chantier::ThreadPool do
     end
   end
   
+  context 'with failures' do
+    it 'raises after 4 failures' do
+      under_test = described_class.new(num_workers=3, max_failures: '4')
+      expect {
+        15.times do 
+          under_test.fork_task { raise "I am such a failure" }
+        end
+      }.to raise_error('Reached error limit of 4 (last error was #<RuntimeError: I am such a failure>)')
+    end
+    
+    it 'runs through the jobs if max_failures is not given' do
+      under_test = described_class.new(num_workers=3)
+      7.times {
+        under_test.fork_task { raise "I am such a failure" }
+      }
+      under_test.block_until_complete!
+      expect(true).to eq(true), "Should have gotten to this assertion without the Pool blocking"
+    end
+  end
+  
   it 'gets instantiated with the given number of slots' do
     described_class.new(10)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chantier
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-03 00:00:00.000000000 Z
+date: 2014-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec