chantier 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 733102f569568fbb2f2953e74298840e6cddd625
-  data.tar.gz: 0753643bc032abc303a72f212dab29f7691338ec
+  metadata.gz: 335d81b6d4f243e4d8516d63f27444fc786f1b43
+  data.tar.gz: a4918eb5ca7855e60056abef9441735ee63e474e
 SHA512:
-  metadata.gz: 8b34b053dba7b457a0d4561c5fff71f14a8fa7e96314c0c9d209c539f874255da769ecd89b1fd96e4e8d1011c4dce4c4ef9b30ddbdd67f9ce8147e3396b93193
-  data.tar.gz: dd1101a603f7f83a28af4ebfd74734daadeecadeeef069987ced8947d5a73b5b527fda0a8f5040bd47409ecd39972b5a8d2e6b502f8f8813f942887fb160dff3
+  metadata.gz: d1fa0022f0fa6d41f715ba08677cb4a8ad6863b9d2da73ccee86d42612c1693a56c92bf61abe7a427d51a46d13eae10242a5d5bd250ba18b6e46a35efed1854d
+  data.tar.gz: b3a59f59fdf5d17bfcc62ee2772f9a15c835dce742e6e65daf1ab28478a45fd2d19dd409ea4582a9f3b951e18f942948e3e27a1ba1e5ce9007b39f91eb18d15d

data/chantier.gemspec

@@ -2,14 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: chantier 0.0.3 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "chantier"
-  s.version = "0.0.2"
+  s.version = "0.0.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2014-07-15"
+  s.date = "2014-08-02"
   s.description = " Process your jobs in parallel with a simple table of processes or threads "
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [
@@ -26,15 +28,16 @@ Gem::Specification.new do |s|
     "chantier.gemspec",
     "lib/chantier.rb",
     "lib/process_pool.rb",
+    "lib/process_pool_with_kill.rb",
     "lib/thread_pool.rb",
     "spec/process_pool_spec.rb",
+    "spec/process_pool_with_kill_spec.rb",
     "spec/spec_helper.rb",
     "spec/thread_pool_spec.rb"
   ]
   s.homepage = "http://github.com/julik/chantier"
   s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.3"
+  s.rubygems_version = "2.2.2"
   s.summary = "Dead-simple worker table based multiprocessing/multithreading"
 
   if s.respond_to? :specification_version then

data/lib/chantier.rb

@@ -1,5 +1,6 @@
 module Chantier
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
   require_relative 'process_pool'
+  require_relative 'process_pool_with_kill'
   require_relative 'thread_pool'
 end

data/lib/process_pool.rb

@@ -26,11 +26,6 @@
 #
 # Can be rewritten using Threads if operation on JVM/Rubinius will be feasible.
 class Chantier::ProcessPool
-  # Kill the spawned processes after at most X seconds
-  # KILL_AFTER_SECONDS = 60 * 2
-  
-  # http://linuxman.wikispaces.com/killing+me+softly
-  # TERMINATION_SIGNALS = %w( TERM HUP INT QUIT PIPE KILL )
   
   # The manager uses loops in a few places. By doing a little sleep()
   # in those loops we can yield process control back to the OS which brings
@@ -97,6 +92,8 @@ class Chantier::ProcessPool
       @semaphore.synchronize { @pids[destination_slot_idx] = nil }
     end
     
+    return task_pid
+    
     # Dispatch the killer thread which kicks in after KILL_AFTER_SECONDS.
     # Note that we do not manage the @pids table here because once the process
     # gets terminated it will bounce back to the standard wait() above.

data/lib/process_pool_with_kill.rb

@@ -0,0 +1,52 @@
+# Allows you to spin off a pool of subprocesses that is not larger than X, and
+# maintains a pool of those proceses (same as ProcessPool). Will also forcibly quit
+# those processes after a certain period to ensure they do not hang
+#
+#   manager = ProcessPoolWithKill.new(slots = 4, kill_after = 5) # seconds
+#   jobs_hose.each_job do | job |
+#     # this call will block until a slot becomes available
+#     manager.fork_task do # this block runs in a subprocess
+#       Churner.new(job).churn
+#     end
+#     manager.still_running? # => most likely "true"
+#   end
+#   
+#   manager.block_until_complete! #=> Will block until all the subprocesses have terminated
+class Chantier::ProcessPoolWithKill < Chantier::ProcessPool
+  
+  # http://linuxman.wikispaces.com/killing+me+softly
+  TERMINATION_SIGNALS = %w( TERM HUP INT QUIT PIPE KILL )
+  
+  DEFAULT_KILL_TIMEOUT = 60
+  def initialize(num_procs, kill_after_seconds = DEFAULT_KILL_TIMEOUT)
+    @kill_after_seconds = kill_after_seconds.to_f
+    super(num_procs)
+  end
+  
+  # Run the given block in a forked subprocess. This method will block
+  # the thread it is called from until a slot in the process table
+  # becomes free. Once that happens, the given block will be forked off
+  # and the method will return.
+  def fork_task(&blk)
+    task_pid = super
+    Thread.abort_on_exception = true
+    # Dispatch the killer thread which kicks in after KILL_AFTER_SECONDS.
+    # Note that we do not manage the @pids table here because once the process
+    # gets terminated it will bounce back to the standard wait() above.
+    Thread.new do
+      sleep @kill_after_seconds
+      TERMINATION_SIGNALS.each do | sig |
+        puts "Dispatching #{sig}"
+        begin
+          Process.kill(sig, task_pid)
+          sleep 1 # Give it some time to react
+        rescue Errno::ESRCH
+          # It has already quit, nothing to do
+        end
+      end
+    end
+    
+    return task_pid
+  end
+  
+end

data/spec/process_pool_spec.rb

@@ -55,10 +55,13 @@ describe Chantier::ProcessPool do
     
     it 'processes 1 file' do
       filename = @files[0]
-      manager.fork_task do
+      pid = manager.fork_task do
         sleep(0.05 + (rand / 10))
         File.open(filename, "wb"){|f| f.write("Worker completed") }
       end
+      
+      expect(pid).to be_kind_of(Fixnum)
+      
       manager.block_until_complete!
       
       expect(File.read(filename)).to eq('Worker completed')

data/spec/process_pool_with_kill_spec.rb

@@ -0,0 +1,125 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Chantier::ProcessPoolWithKill do
+  
+  before(:each) do
+    @files = (0...20).map do
+      SecureRandom.hex(12).tap { |filename| FileUtils.touch(filename) }
+    end
+  end
+  
+  after(:each) do
+    @files.map(&File.method(:unlink))
+  end
+  
+  context '#map_fork' do
+    let(:manager) { described_class.new(5) }
+    
+    it 'processes multiple files' do
+      
+      data_chunks = (0..10).map{|e| Digest::SHA1.hexdigest(e.to_s) }
+      
+      expect(manager).not_to be_still_running
+      
+      manager.map_fork(@files) do | filename |
+        sleep(0.05 + (rand / 10))
+        File.open(filename, "wb"){|f| f.write("Worker completed for #{filename}") }
+      end
+      
+      expect(manager).not_to be_still_running
+      
+      @files.each do | filename |
+        expect(File.read(filename)).to eq("Worker completed for #{filename}")
+      end
+    end
+  end
+  
+  context 'with 0 concurrent slots' do
+    it 'raises an exception' do
+      expect {
+        described_class.new(0)
+      }.to raise_error(RuntimeError, 'Need at least 1 slot, given 0')
+      
+      expect {
+        described_class.new(-1)
+      }.to raise_error(RuntimeError, 'Need at least 1 slot, given -1')
+    end
+  end
+  
+  context 'with 1 slot' do
+    let(:manager) { described_class.new(1) }
+    
+    it 'processes 1 file' do
+      filename = @files[0]
+      pid = manager.fork_task do
+        sleep(0.05 + (rand / 10))
+        File.open(filename, "wb"){|f| f.write("Worker completed") }
+      end
+      
+      expect(pid).to be_kind_of(Fixnum)
+      
+      manager.block_until_complete!
+      
+      expect(File.read(filename)).to eq('Worker completed')
+    end
+    
+    it 'processes multiple files' do
+      expect(manager).not_to be_still_running
+      
+      @files.each do | filename |
+        manager.fork_task do
+          sleep(0.05 + (rand / 10))
+          File.open(filename, "wb"){|f| f.write("Worker completed for #{filename}") }
+        end
+      end
+      
+      expect(manager).to be_still_running
+      
+      manager.block_until_complete!
+      
+      @files.each do | filename |
+        expect(File.read(filename)).to eq("Worker completed for #{filename}")
+      end
+    end
+    
+    it 'forcibly quites a process that is hung for too long' do
+      manager_with_short_timeout = described_class.new(1, timeout=0.4)
+      
+      filename = SecureRandom.hex(22)
+      manager_with_short_timeout.fork_task do
+        10.times do
+          sleep 1 # WAY longer than the timeout
+        end
+        File.open(filename, "wb") {|f| f.write("Should never happen")}
+      end
+      
+      manager_with_short_timeout.block_until_complete!
+      expect(File.exist?(filename)).to eq(false)
+    end
+  end
+  
+  context 'with 5 slots' do
+    let(:manager) { described_class.new(5) }
+    
+    it 'processes multiple files' do
+      
+      expect(manager).not_to be_still_running
+      
+      @files.each do | filename |
+        manager.fork_task do
+          sleep(0.05 + (rand / 10))
+          File.open(filename, "wb"){|f| f.write("Worker completed for #{filename}") }
+        end
+      end
+      
+      expect(manager).to be_still_running
+      
+      manager.block_until_complete!
+      
+      @files.each do | filename |
+        expect(File.read(filename)).to eq("Worker completed for #{filename}")
+      end
+    end
+  end
+end
+

metadata

@@ -1,86 +1,86 @@
 --- !ruby/object:Gem::Specification
 name: chantier
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-15 00:00:00.000000000 Z
+date: 2014-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.9'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.12'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: jeweler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.1
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ' Process your jobs in parallel with a simple table of processes or threads '
+description: " Process your jobs in parallel with a simple table of processes or threads "
 email: me@julik.nl
 executables: []
 extensions: []
@@ -88,8 +88,8 @@ extra_rdoc_files:
 - LICENSE.txt
 - README.rdoc
 files:
-- .document
-- .rspec
+- ".document"
+- ".rspec"
 - Gemfile
 - LICENSE.txt
 - README.rdoc
@@ -97,8 +97,10 @@ files:
 - chantier.gemspec
 - lib/chantier.rb
 - lib/process_pool.rb
+- lib/process_pool_with_kill.rb
 - lib/thread_pool.rb
 - spec/process_pool_spec.rb
+- spec/process_pool_with_kill_spec.rb
 - spec/spec_helper.rb
 - spec/thread_pool_spec.rb
 homepage: http://github.com/julik/chantier
@@ -111,17 +113,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Dead-simple worker table based multiprocessing/multithreading