process_pool 0.1.0

data/.gitignore

@@ -0,0 +1,5 @@
+.idea/*
+vendor/*
+bin/*
+tmp/*
+scratch_directory/*

data/Gemfile

@@ -0,0 +1,8 @@
+disable_system_gems
+
+gem 'json'
+
+only :test do
+  gem 'shoulda'
+  gem 'mocha'
+end

data/LICENSE

@@ -0,0 +1,21 @@
+== MIT License
+
+Copyright (c) 2010, Adam Pohorecki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README

@@ -0,0 +1,21 @@
+Process Pool is something I created to replace a thread pool that had major performance problems caused by Ruby's (MRI)
+poor thread implementation.
+
+The basic usage is:
+
+class SomeTask
+  def initialize(x, y, z)
+  end
+
+  def run
+  end
+end
+
+pool = ProcessPool.new(5) # first constructor argument is a number of worker processes
+pool.schedule SomeTask, 1, 2, 3 # adds task SomeTask.new(1,2,3) to the queue (instantiation happens just before calling run())
+pool.start # starts the workers
+pool.schedule SomeTask, 3, 2, 1 # tasks can be added also after calling start
+pool.shutdown # waits for all the tasks in the queue to be processed and kills all the worker processes
+
+For now there is only one, very naive, queue implementation, which uses file locking and JSON storage format. I plan on adding
+some other (better) queue backend soon.

data/Rakefile

@@ -0,0 +1,13 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "process_pool"
+    gemspec.summary = "ProcessPool with interchangeable job queue backends for Ruby"
+    gemspec.email = "adam@pohorecki.pl"
+    gemspec.homepage = "http://github.com/psyho/process_pool"
+    gemspec.authors = ["Adam Pohorecki"]
+    gemspec.add_dependency 'json'
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/convert_to_should_syntax

@@ -0,0 +1,3 @@
+#!/home/psyho/.rvm/ruby-1.8.7-p174/bin/ruby
+require File.join(File.dirname(__FILE__), "../vendor/gems/environment")
+load File.join(File.dirname(__FILE__), "../vendor/gems/gems/shoulda-2.10.3/bin/convert_to_should_syntax")

data/bin/edit_json.rb

@@ -0,0 +1,3 @@
+#!/home/psyho/.rvm/ruby-1.8.7-p174/bin/ruby
+require File.join(File.dirname(__FILE__), "../vendor/gems/environment")
+load File.join(File.dirname(__FILE__), "../vendor/gems/gems/json-1.2.0/bin/edit_json.rb")

data/bin/flay

@@ -0,0 +1,3 @@
+#!/home/psyho/.rvm/ruby-1.8.7-p174/bin/ruby
+require File.join(File.dirname(__FILE__), "../vendor/gems/environment")
+load File.join(File.dirname(__FILE__), "../vendor/gems/gems/flay-1.4.0/bin/flay")

data/bin/flog

@@ -0,0 +1,3 @@
+#!/home/psyho/.rvm/ruby-1.8.7-p174/bin/ruby -w
+require File.join(File.dirname(__FILE__), "../vendor/gems/environment")
+load File.join(File.dirname(__FILE__), "../vendor/gems/gems/flog-2.2.0/bin/flog")

data/bin/prettify_json.rb

@@ -0,0 +1,3 @@
+#!/home/psyho/.rvm/ruby-1.8.7-p174/bin/ruby
+require File.join(File.dirname(__FILE__), "../vendor/gems/environment")
+load File.join(File.dirname(__FILE__), "../vendor/gems/gems/json-1.2.0/bin/prettify_json.rb")

data/bin/rake

@@ -0,0 +1,3 @@
+#!/home/psyho/.rvm/ruby-1.8.7-p174/bin/ruby
+require File.join(File.dirname(__FILE__), "../vendor/gems/environment")
+load File.join(File.dirname(__FILE__), "../vendor/gems/gems/rake-0.8.7/bin/rake")

data/bin/rcov

@@ -0,0 +1,3 @@
+#!/home/psyho/.rvm/ruby-1.8.7-p174/bin/ruby
+require File.join(File.dirname(__FILE__), "../vendor/gems/environment")
+load File.join(File.dirname(__FILE__), "../vendor/gems/gems/rcov-0.9.7.1/bin/rcov")

data/bin/ruby_parse

@@ -0,0 +1,3 @@
+#!/home/psyho/.rvm/ruby-1.8.7-p174/bin/ruby -s
+require File.join(File.dirname(__FILE__), "../vendor/gems/environment")
+load File.join(File.dirname(__FILE__), "../vendor/gems/gems/ruby_parser-2.0.4/bin/ruby_parse")

data/bin/rubyforge

@@ -0,0 +1,3 @@
+#!/home/psyho/.rvm/ruby-1.8.7-p174/bin/ruby
+require File.join(File.dirname(__FILE__), "../vendor/gems/environment")
+load File.join(File.dirname(__FILE__), "../vendor/gems/gems/rubyforge-2.0.3/bin/rubyforge")

data/bin/sow

@@ -0,0 +1,3 @@
+#!/home/psyho/.rvm/ruby-1.8.7-p174/bin/ruby -ws
+require File.join(File.dirname(__FILE__), "../vendor/gems/environment")
+load File.join(File.dirname(__FILE__), "../vendor/gems/gems/hoe-2.5.0/bin/sow")

data/lib/init.rb

@@ -0,0 +1,6 @@
+require 'rubygems'
+require 'json'
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'simple_logger'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'simple_queue'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'process_pool'))

data/lib/process_pool.rb

@@ -0,0 +1,98 @@
+class ProcessPool
+
+  class InvalidStateError < StandardError;
+  end
+
+  attr_reader :workers_count
+
+  def initialize(workers_count, queue = SimpleQueue.create, logger = SimpleLogger.new)
+    self.state = :stopped
+    self.logger = logger
+    self.workers_count = workers_count
+    self.queue = queue
+    self.worker_pids = []
+  end
+
+  def schedule(job_class, *args)
+    raise InvalidStateError.new('Can not add more jobs after shut down was called') if is_shutdown?
+    logger.debug("Scheduling task #{job_class}(#{args})")
+    push_task(job_class, args)
+  end
+
+  def start
+    raise InvalidStateError.new('Can not start a pool more than once') unless is_stopped?
+    logger.info("Starting process pool")
+    self.state = :running
+
+    workers_count.times do
+      pid = fork do
+        child_queue = get_child_queue()
+        while true
+          task_class, args = child_queue.pop
+          begin
+            task = get_task_class(task_class).new(*args)
+            task.run
+          rescue => e
+            logger.warn("Exception occurred while executing task #{task_class}(#{args}): #{e}")
+          end
+        end
+      end
+      self.worker_pids << pid
+    end
+  end
+
+  def shutdown
+    raise InvalidStateError.new('Can not shut down pool that is not running') unless is_running?
+    logger.info("Shutting down process pool")
+    self.state = :shutdown
+
+    workers_count.times do
+      push_task(EndTask, [])
+    end
+
+    worker_pids.each do |pid|
+      Process.wait(pid)
+    end
+  end
+
+  def is_running?
+    return state == :running
+  end
+
+  def is_stopped?
+    return state == :stopped
+  end
+
+  def is_shutdown?
+    return state == :shutdown
+  end
+
+  protected
+
+  attr_accessor :state, :logger, :queue, :worker_pids
+  attr_writer :workers_count
+
+  def push_task(job_class, args)
+    queue.push([job_class.name.to_s, args])
+  end
+
+  def get_child_queue
+    queue.class.get(queue.uri)
+  end
+
+  # this is taken from ActiveSupport (String#constantize)
+  def get_task_class(class_name)
+    unless /\A(?:::)?([A-Z]\w*(?:::[A-Z]\w*)*)\z/ =~ class_name
+      raise NameError, "#{class_name.inspect} is not a valid constant name!"
+    end
+
+    Object.module_eval("::#{$1}", __FILE__, __LINE__)
+  end
+
+  class EndTask
+    def run
+      exit(0)
+    end
+  end
+
+end

data/lib/simple_logger.rb

@@ -0,0 +1,19 @@
+class SimpleLogger
+  LEVELS = [:debug, :info, :warn, :error, :fatal]
+
+  attr_accessor :level
+
+  def initialize(level = :info)
+    self.level = level
+  end
+
+  LEVELS.each do |level|
+    define_method(level) do |msg|
+      idx = LEVELS.index(level)
+      if idx >= LEVELS.index(self.level)
+        puts "Process #{Process.pid}: [#{level.to_s.upcase}] #{msg}"
+      end
+    end
+  end
+
+end

data/lib/simple_queue.rb

@@ -0,0 +1,93 @@
+require 'rubygems'
+require 'json'
+require 'tempfile'
+
+class SimpleQueue
+
+  attr_reader :uri
+
+  def close
+    File.delete(uri)
+  end
+
+  def push(value)
+    with_queue_file do |file|
+      contents = file.read
+      queue = JSON.parse(contents)
+      queue.push(value)
+      store_queue(file, queue)
+    end
+  end
+
+  def pop
+    result = nil
+    queue_empty = true
+
+    while queue_empty
+      queue_empty, result = pop_nowait()
+      sleep(0.1) if queue_empty
+    end
+    
+    return result
+  end
+
+  def size
+    contents = ''
+    with_queue_file do |file|
+      contents = file.read
+    end
+    return JSON.parse(contents).size
+  end
+
+  def self.create
+    file = Tempfile.new('simple_queue')
+    file.puts [].to_json
+    uri = file.path
+    file.close
+    return new(uri)
+  end
+
+  def self.get(uri)
+    raise ArgumentError.new("Queue file must exist: #{uri}") unless File.exists?(uri)
+    return new(uri)
+  end
+
+  protected
+
+  attr_writer :uri
+
+  def initialize(uri)
+    self.uri = uri
+  end
+
+  def with_queue_file(&block)
+    file = File.new(uri, 'r+')
+    if file.flock(File::LOCK_EX)
+      block.call(file)
+    end
+    file.close
+  end
+
+  def store_queue(file, queue)
+    file.truncate(0)
+    file.seek(0)
+
+    file.puts queue.to_json
+  end
+
+  def pop_nowait
+    queue_empty = true
+    result = nil
+    with_queue_file do |file|
+      contents = file.read
+      queue = JSON.parse(contents)
+      unless queue.empty?
+        queue_empty = false
+        result = queue.shift
+        store_queue(file, queue)
+      end
+    end
+    return queue_empty, result
+  end
+
+end

data/test/process_pool_test.rb

@@ -0,0 +1,218 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'test_helper')) 
+
+class SampleQueue
+
+  attr_accessor :data
+
+  def initialize
+    self.data = []
+  end
+
+  def uri
+    'test-queue'
+  end
+
+  def push(value)
+    self.data << value
+  end
+
+  def pop
+    self.data.shift
+  end
+
+  def size
+    self.data.size
+  end
+
+  def self.create
+    new
+  end
+
+  def self.get(uri)
+    new
+  end
+
+end
+
+class SampleTask
+
+  attr_accessor :args
+
+  def initialize(*args)
+    self.args = args
+  end
+
+  def run
+    return args
+  end
+
+end
+
+class ExceptionTask
+  def run
+    raise ArgumentError.new("something went wrong")
+  end
+end
+
+class WrongArgumentsTask
+  def initialize(x, y, z)
+  end
+
+  def run    
+  end
+end
+
+class ProcessPoolTest < Test::Unit::TestCase
+
+  context "state" do
+    setup do
+      @pool = ProcessPool.new(10, SampleQueue.new, SimpleLogger.new(:debug))
+      @pool.stubs(:fork => 1)
+      Process.stubs(:wait => 0)
+    end
+
+    should "be stopped after initialization" do
+      assert @pool.is_stopped?
+      assert !@pool.is_running?
+      assert !@pool.is_shutdown?
+    end
+
+    should "be running after start" do
+      @pool.start
+      assert !@pool.is_stopped?
+      assert @pool.is_running?
+      assert !@pool.is_shutdown?
+    end
+
+    should "be shutdown after shutdown" do
+      @pool.start
+      @pool.shutdown
+      assert !@pool.is_stopped?
+      assert !@pool.is_running?
+      assert @pool.is_shutdown?
+    end
+
+    context "invalid actions" do
+      should "raise InvalidStateError when calling shutdown on a not started pool" do
+        assert_raises ProcessPool::InvalidStateError do
+          @pool.shutdown
+        end
+      end
+
+      should "raise InvalidStateError when calling start twice" do
+        @pool.start
+        assert_raises ProcessPool::InvalidStateError do
+          @pool.start
+        end
+      end
+
+      should "raise InvalidStateError when calling shutdown twice" do
+        @pool.start
+        @pool.shutdown
+        assert_raises ProcessPool::InvalidStateError do
+          @pool.shutdown
+        end
+      end
+
+      should "raise InvalidStateError when calling schedule on a shutdown pool" do
+        @pool.start
+        @pool.shutdown
+        assert_raises ProcessPool::InvalidStateError do
+          @pool.schedule(SampleTask)
+        end
+      end
+    end
+  end
+
+  context :schedule do
+    setup do
+      @queue = SampleQueue.new
+      @pool = ProcessPool.new(10, @queue, SimpleLogger.new(:debug))
+      @pool.stubs(:fork => 1)
+    end
+
+    should "add jobs to the queue" do
+      assert_equal 0, @queue.size
+      @pool.schedule(SampleTask, 1, 2, 3)
+      assert_equal 1, @queue.size
+      assert_equal ["SampleTask", [1, 2, 3]], @queue.pop
+    end
+  end
+
+  context :start do
+    setup do
+      @queue = SampleQueue.new
+      @workers_count = 10
+      @pool = ProcessPool.new(@workers_count, @queue, SimpleLogger.new(:debug))
+    end
+
+    should "fork workers_count workers" do
+      @pool.expects(:fork).times(@workers_count).returns(0)
+      @pool.start
+    end
+  end
+
+  context :shutdown do
+    setup do
+      @queue = SampleQueue.new
+      @pool = ProcessPool.new(3, @queue, SimpleLogger.new(:debug)) # no workers so that nothing processes the queue
+      @pool.schedule(SampleTask)
+      @pool.stubs(:fork => 1)
+      @pool.start
+    end
+
+    should "schedule same number od EndTasks as worker_count" do
+      Process.stubs(:wait => 0)
+      @pool.shutdown
+      assert_equal 4, @queue.size
+      assert_equal "SampleTask", @queue.data[0].first
+      (1..3).each do |n|
+        assert_equal "ProcessPool::EndTask", @queue.data[n].first
+      end
+    end
+
+    should "wait on all of the worker processes to end" do
+      Process.expects(:wait).with(1).times(3).returns(0)
+      @pool.shutdown
+    end
+  end
+
+  context "with default queue and two workers" do
+    setup do
+      @pool = ProcessPool.new(1)
+      2.times { |n| @pool.schedule(SampleTask, n) }
+    end
+
+    should "empty the queue before returning from shutdown" do
+      assert_equal 2, @pool.send(:queue).size
+      @pool.start
+      @pool.shutdown
+      assert_equal 0, @pool.send(:queue).size
+    end
+
+    should "empty the queue even if some tasks result in exception" do
+      @pool.schedule(ExceptionTask)
+      assert_equal 3, @pool.send(:queue).size
+      @pool.start
+      @pool.shutdown
+      assert_equal 0, @pool.send(:queue).size
+    end
+
+    should "empty the queue even if some tasks can not be loaded" do
+      @pool.schedule(String)
+      assert_equal 3, @pool.send(:queue).size
+      @pool.start
+      @pool.shutdown
+      assert_equal 0, @pool.send(:queue).size
+    end
+
+    should "empty the queue even if some tasks can not be initialized" do
+      @pool.schedule(WrongArgumentsTask, 1)
+      assert_equal 3, @pool.send(:queue).size
+      @pool.start
+      @pool.shutdown
+      assert_equal 0, @pool.send(:queue).size
+    end
+  end
+
+end

data/test/simple_queue_test.rb

@@ -0,0 +1,101 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'test_helper'))
+
+class SimpleQueueTest < Test::Unit::TestCase
+
+  def setup
+    @queue = SimpleQueue.create
+  end
+
+  def teardown
+    @queue.close if @queue
+  end
+
+  context :create do
+    should "return a queue" do
+      assert @queue.is_a?(SimpleQueue)
+    end
+
+    should "return queues with different URIs every time" do
+      new_queue = SimpleQueue.create
+      new_queue.close
+      assert new_queue.uri != @queue.uri
+    end
+
+    should "return a queue that is empty" do
+      assert_equal 0, @queue.size
+    end
+  end
+
+  context :get do
+    setup do
+      @uri = @queue.uri
+    end
+
+    should "raise an ArgumentError if queue does not exist" do
+      assert_raises(ArgumentError) { SimpleQueue.get('not existing') }
+    end
+
+    should "return a queue if it does exist" do
+      queue = SimpleQueue.get(@uri)
+      assert queue
+      assert_equal @uri, queue.uri
+    end
+  end
+
+  context :pop do
+    context "on a queue that contains something" do
+      setup do
+        @queue.push('hello')
+      end
+
+      context "pop" do
+        setup do
+          @result = @queue.pop
+        end
+
+        should "return 'hello'" do
+          assert_equal 'hello', @result
+        end
+
+        should_change('queue size', :by => -1) { @queue.size }
+      end
+    end
+
+    context 'on an empty queue' do
+      context 'pop' do
+        should 'block until something gets pushed' do
+          pid = fork do
+            queue = SimpleQueue.get(@queue.uri)
+            value = queue.pop
+            exit(value || 0)
+          end
+          sleep(0.5) # give the child process some time to start
+          @queue.push(19) 
+          pid, status = Process.waitpid2(pid)
+          assert_equal 19, status.exitstatus
+        end
+      end
+    end
+  end
+
+  context :push do
+    context "on a queue that contains something" do
+      setup do
+        @queue.push('hello')
+      end
+
+      should_change('queue size', :by => 1) { @queue.size }
+    end
+  end
+
+  should "work on a FIFO basis" do
+    elements = [1, 2, 3, 4, 5]
+    elements.each { |e| @queue.push(e) }
+    popped = []
+    elements.size.times do
+      popped << @queue.pop
+    end
+    assert_equal elements, popped
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,5 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'vendor', 'gems', 'environment'))
+
+Bundler.require_env :test
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'init'))

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification 
+name: process_pool
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Adam Pohorecki
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-01-17 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: 
+email: adam@pohorecki.pl
+executables: 
+- convert_to_should_syntax
+- rake
+- rubyforge
+- flog
+- ruby_parse
+- flay
+- rcov
+- edit_json.rb
+- sow
+- prettify_json.rb
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README
+files: 
+- .gitignore
+- Gemfile
+- LICENSE
+- README
+- Rakefile
+- VERSION
+- lib/init.rb
+- lib/process_pool.rb
+- lib/simple_logger.rb
+- lib/simple_queue.rb
+- test/process_pool_test.rb
+- test/simple_queue_test.rb
+- test/test_helper.rb
+has_rdoc: true
+homepage: http://github.com/psyho/process_pool
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: ProcessPool with interchangeable job queue backends for Ruby
+test_files: 
+- test/test_helper.rb
+- test/simple_queue_test.rb
+- test/process_pool_test.rb