process_pool 0.1.2 → 0.1.3

data/Gemfile

@@ -1,6 +1,7 @@
 disable_system_gems
 
 gem 'json'
+gem 'jeweler'
 
 only :test do
   gem 'shoulda'

data/TODO

@@ -0,0 +1,12 @@
+Extensions stuff:
+- around extensions
+- handling :stop
+- helpers for anonymous extensions
+- a couple of generic extensions:
+  - restarting active record connections on worker startup
+  - benchmarking
+  - restarting failed tasks
+
+Queue:
+  - posix shared memory or message queue backend
+  - redis backend

data/VERSION

@@ -1 +1 @@
-0.1.2
+0.1.3

data/lib/anonymous_extension.rb

@@ -0,0 +1,20 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), 'init')
+
+class AnonymousExtension < BaseWorkerExtension
+
+  SUPPORTED_EXTENSION_METHODS = [:before, :after, :around, :startup, :shutdown]
+
+  def initialize(method, &block)
+    raise ArgumentError.new("Unknown method: #{method}") unless SUPPORTED_EXTENSION_METHODS.include?(method)
+
+    @method = method
+    @block = lambda(&block)
+  end
+
+  SUPPORTED_EXTENSION_METHODS.each do |method_name|
+    define_method(method_name) do |*args|      
+      @block.call(*args) if method_name == @method      
+    end
+  end
+
+end

data/lib/base_worker_extension.rb

@@ -0,0 +1,35 @@
+# base class for process pool extensions
+#
+# it is not required that the extensions inherit from this class,
+# it's here mostly for convenience and documentation sake 
+class BaseWorkerExtension
+
+  attr_accessor :logger, :process_pool
+
+  # will be called before task gets executed
+  # thowing :stop will halt the task execution process,
+  # no other filters will be executed for this task
+  def before(task)
+  end
+
+  # will be called after task gets executed
+  def after(task, result)
+  end
+
+  # will be called before task gets executed
+  # around is responsible for calling task.run,
+  # failing to do so might result in unpredictable behavior
+  # and failing to run a ProcessPool::EndTask will result in
+  # the process never ending
+  def around(task)
+  end
+
+  # will be called on worker startup, before executing any tasks
+  def startup
+  end
+
+  # will be called on worker shutdown
+  def shutdown    
+  end
+
+end

data/lib/init.rb

@@ -4,3 +4,5 @@ require 'json'
 require File.expand_path(File.join(File.dirname(__FILE__), 'simple_logger'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'simple_queue'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'process_pool'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'base_worker_extension'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'anonymous_extension'))

data/lib/process_pool.rb

@@ -13,14 +13,25 @@ class ProcessPool
     self.workers_count = workers_count
     self.queue = queue
     self.worker_pids = []
+    self.extensions = []
   end
 
   def schedule(job_class, *args)
     raise InvalidStateError.new('Can not add more jobs after shut down was called') if is_shutdown?
-    logger.debug("Scheduling task #{job_class}(#{args})")
+    logger.debug("Scheduling task #{job_class}(#{args.collect{ |a| a.inspect }.join(', ')})")
     push_task(job_class, args)
   end
 
+  def register_extension(extension)
+    raise InvalidStateError.new('Can not register extensions once the pool is started.') unless is_stopped?
+    raise ArgumentError.new('extension can not be nil') unless extension
+
+    extension.process_pool = self if extension.respond_to?(:process_pool=)
+    extension.logger = logger if extension.respond_to?(:logger=)
+
+    extensions << extension
+  end
+
   def start
     raise InvalidStateError.new('Can not start a pool more than once') unless is_stopped?
     logger.info("Starting process pool")
@@ -28,12 +39,16 @@ class ProcessPool
 
     workers_count.times do
       pid = fork do
+        run_extensions(:startup)
+        at_exit { run_extensions(:shutdown) }
         child_queue = get_child_queue()
         while true
           task_class, args = child_queue.pop
           begin
             task = get_task_class(task_class).new(*args)
-            task.run
+            run_extensions(:before, task) unless task.is_a?(EndTask)
+            result = task.run
+            run_extensions(:after, task, result)
           rescue => e
             logger.warn("Exception occurred while executing task #{task_class}(#{args}): #{e}")
           end
@@ -73,7 +88,7 @@ class ProcessPool
 
   protected
 
-  attr_accessor :state, :logger, :queue, :worker_pids
+  attr_accessor :state, :logger, :queue, :worker_pids, :extensions
   attr_writer :workers_count
 
   def push_task(job_class, args)
@@ -93,6 +108,14 @@ class ProcessPool
     Object.module_eval("::#{$1}", __FILE__, __LINE__)
   end
 
+  def run_extensions(method, *args)
+    extensions.each do |extension|
+      if extension.respond_to?(method)
+        extension.send(method, *args)
+      end
+    end
+  end
+
   class EndTask
     def run
       exit(0)

data/lib/simple_queue.rb

@@ -43,10 +43,14 @@ class SimpleQueue
 
   def self.create
     file = Tempfile.new('simple_queue')
-    file.puts [].to_json
-    uri = file.path
-    file.close
-    return new(uri)
+    path = File.expand_path(file.path)
+    file.close!
+
+    File.open(path, 'w+') do |f|
+      f.puts [].to_json  
+    end
+
+    return new(path)
   end
 
   def self.get(uri)

data/process_pool.gemspec

@@ -5,15 +5,16 @@
 
 Gem::Specification.new do |s|
   s.name = %q{process_pool}
-  s.version = "0.1.2"
+  s.version = "0.1.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Adam Pohorecki"]
-  s.date = %q{2010-01-20}
+  s.date = %q{2010-02-18}
   s.email = %q{adam@pohorecki.pl}
   s.extra_rdoc_files = [
     "LICENSE",
-     "README"
+     "README",
+     "TODO"
   ]
   s.files = [
     ".gitignore",
@@ -21,12 +22,16 @@ Gem::Specification.new do |s|
      "LICENSE",
      "README",
      "Rakefile",
+     "TODO",
      "VERSION",
+     "lib/anonymous_extension.rb",
+     "lib/base_worker_extension.rb",
      "lib/init.rb",
      "lib/process_pool.rb",
      "lib/simple_logger.rb",
      "lib/simple_queue.rb",
      "process_pool.gemspec",
+     "test/anonymous_extension_test.rb",
      "test/process_pool_test.rb",
      "test/simple_queue_test.rb",
      "test/test_helper.rb"
@@ -37,7 +42,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.5}
   s.summary = %q{ProcessPool with interchangeable job queue backends for Ruby}
   s.test_files = [
-    "test/process_pool_test.rb",
+    "test/anonymous_extension_test.rb",
+     "test/process_pool_test.rb",
      "test/simple_queue_test.rb",
      "test/test_helper.rb"
   ]

data/test/anonymous_extension_test.rb

@@ -0,0 +1,41 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'test_helper'))
+
+class AnonymousExtensionTest < Test::Unit::TestCase
+
+  should "raise ArgumentError when called with an unsupported method" do
+    assert_raises ArgumentError do
+      AnonymousExtension.new(:unsupported_method) {}
+    end
+  end
+
+  AnonymousExtension::SUPPORTED_EXTENSION_METHODS.each do |method_name|
+    should "not raise anything when called with #{method_name}" do
+      assert_nothing_raised do
+        AnonymousExtension.new(method_name) {}
+      end
+    end
+
+    context "with method #{method_name}" do
+      setup do
+        @value = 0
+        @ext = AnonymousExtension.new(method_name) { |x| @value = x }
+      end
+
+      should "execute the block when right method is called" do
+        @ext.send(method_name, 1)
+        assert_equal 1, @value
+      end
+
+      AnonymousExtension::SUPPORTED_EXTENSION_METHODS.each do |other_method|
+        next if method_name == other_method
+
+        should "not execute the block when #{other_method} is called" do
+          @ext.send(other_method, 1)
+          assert_equal 0, @value
+        end
+      end
+    end
+
+  end
+
+end

data/test/process_pool_test.rb

@@ -65,6 +65,21 @@ class WrongArgumentsTask
   end
 end
 
+class WriteToFileTask
+
+  def initialize(path, n)
+    @path = path
+    @n = n
+  end
+
+  def run
+    File.open(@path, 'a+') do |file|
+      file.puts "task #{@n}"
+    end
+  end
+
+end
+
 class ProcessPoolTest < Test::Unit::TestCase
 
   context "state" do
@@ -102,6 +117,21 @@ class ProcessPoolTest < Test::Unit::TestCase
         end
       end
 
+      should "raise InvalidStateError when calling register_extension on a started pool" do
+        @pool.start
+        assert_raises ProcessPool::InvalidStateError do
+          @pool.register_extension(BaseWorkerExtension.new)
+        end
+      end
+
+      should "raise InvalidStateError when calling register_extension on a shutdown pool" do
+        @pool.start
+        @pool.shutdown
+        assert_raises ProcessPool::InvalidStateError do
+          @pool.register_extension(BaseWorkerExtension.new)
+        end
+      end
+
       should "raise InvalidStateError when calling start twice" do
         @pool.start
         assert_raises ProcessPool::InvalidStateError do
@@ -158,7 +188,7 @@ class ProcessPoolTest < Test::Unit::TestCase
   context :shutdown do
     setup do
       @queue = SampleQueue.new
-      @pool = ProcessPool.new(3, @queue, SimpleLogger.new(:debug)) # no workers so that nothing processes the queue
+      @pool = ProcessPool.new(3, @queue, SimpleLogger.new(:debug))
       @pool.schedule(SampleTask)
       @pool.stubs(:fork => 1)
       @pool.start
@@ -230,4 +260,185 @@ class ProcessPoolTest < Test::Unit::TestCase
 
   end
 
+  def self.should_write_lines(*lines)
+    should "write lines #{lines.collect{|line| line.inspect}.join(', ')} to file" do
+      text = open(@path).read
+      file_lines = text.split("\n").collect { |line| line.strip }
+      assert_equal lines, file_lines
+    end
+  end
+
+  def write_to_shared_file(str)
+    File.open(@path, 'a+') do |file|
+      file.puts str
+    end
+  end
+
+  context "extensions" do
+    setup do
+      @logger = SimpleLogger.new(:debug)
+      @pool = ProcessPool.new(1, SimpleQueue.create, @logger)
+      @shared_file = Tempfile.new('test')
+      @path = @shared_file.path
+    end
+
+    teardown do
+      @shared_file.close
+    end
+
+    should "raise ArgumentError if called with nil" do
+      assert_raises ArgumentError do
+        @pool.register_extension nil
+      end
+    end
+
+    should "inject process_pool to the extension" do
+      extension = BaseWorkerExtension.new
+      assert_nil extension.process_pool
+      @pool.register_extension(extension)
+      assert_equal @pool, extension.process_pool
+    end
+
+    should "inject logger to the extension" do
+      extension = BaseWorkerExtension.new
+      assert_nil extension.logger
+      @pool.register_extension(extension)
+      assert_equal @logger, extension.logger
+    end
+
+    context "with no extensions" do
+      setup do
+        @pool.schedule(WriteToFileTask, @path, 1)
+        @pool.schedule(WriteToFileTask, @path, 2)
+
+        @pool.start
+        @pool.shutdown
+      end
+
+      should_write_lines "task 1", "task 2"      
+    end
+
+    context "with before extension" do
+      setup do
+        @pool.schedule(WriteToFileTask, @path, 1)
+        @pool.schedule(WriteToFileTask, @path, 2)
+
+        @pool.register_extension AnonymousExtension.new(:before) { |t| write_to_shared_file("before") }
+
+        @pool.start
+        @pool.shutdown
+      end
+
+      should_write_lines "before", "task 1", "before", "task 2"
+    end
+
+    context "with after extension" do
+      setup do
+        @pool.schedule(WriteToFileTask, @path, 1)
+        @pool.schedule(WriteToFileTask, @path, 2)
+
+        @pool.register_extension AnonymousExtension.new(:after) { |t, r| write_to_shared_file("after") }
+
+        @pool.start
+        @pool.shutdown
+      end
+
+      should_write_lines "task 1", "after", "task 2", "after"
+    end
+
+    context "with around extension" do
+      setup do
+        @pool.schedule(WriteToFileTask, @path, 1)
+        @pool.schedule(WriteToFileTask, @path, 2)
+
+        @pool.register_extension AnonymousExtension.new(:around) { |t|
+          write_to_shared_file("around 1")
+          t.run
+          write_to_shared_file("around 2")
+        }
+
+        @pool.start
+        @pool.shutdown
+      end
+
+      should_write_lines "around 1", "task 1", "around 2", "around 1", "task 2", "around 2"
+    end
+
+    context "with multiple around extensions" do
+      setup do
+        @pool.schedule(WriteToFileTask, @path, 1)
+        @pool.schedule(WriteToFileTask, @path, 2)
+
+        @pool.register_extension AnonymousExtension.new(:around) { |t|
+          write_to_shared_file("around a")
+          t.run
+          write_to_shared_file("around b")
+        }
+        
+        @pool.register_extension AnonymousExtension.new(:around) { |t|
+          write_to_shared_file("around 1")
+          t.run
+          write_to_shared_file("around 2")
+        }
+
+        @pool.start
+        @pool.shutdown
+      end
+
+      should_write_lines "around a", "around 1", "task 1", "around 2", "around b", "around a", "around 1", "task 2", "around 2", "around b"
+    end
+
+    context "with startup extension" do
+      setup do
+        @pool.schedule(WriteToFileTask, @path, 1)
+        @pool.schedule(WriteToFileTask, @path, 2)
+
+        @pool.register_extension AnonymousExtension.new(:startup) { write_to_shared_file("startup") }
+
+        @pool.start
+        @pool.shutdown
+      end
+
+      should_write_lines "startup", "task 1", "task 2"
+    end
+
+    context "with shutdown extension" do
+      setup do
+        @pool.schedule(WriteToFileTask, @path, 1)
+        @pool.schedule(WriteToFileTask, @path, 2)
+
+        @pool.register_extension AnonymousExtension.new(:shutdown) { write_to_shared_file("shutdown") }
+
+        @pool.start
+        @pool.shutdown
+      end
+
+      should_write_lines "task 1", "task 2", "shutdown"
+    end
+
+    context "with all kinds of extensions" do
+      setup do
+        @pool.schedule(WriteToFileTask, @path, 1)
+        @pool.schedule(WriteToFileTask, @path, 2)
+
+        @pool.register_extension AnonymousExtension.new(:around) { |t|
+          write_to_shared_file("around 1")
+          t.run
+          write_to_shared_file("around 2")
+        }
+
+        @pool.register_extension AnonymousExtension.new(:before) { |t| write_to_shared_file("before") }
+        @pool.register_extension AnonymousExtension.new(:after) { |t, r| write_to_shared_file("after") }
+
+        @pool.register_extension AnonymousExtension.new(:startup) { write_to_shared_file("startup") }
+        @pool.register_extension AnonymousExtension.new(:shutdown) { write_to_shared_file("shutdown") }
+
+        @pool.start
+        @pool.shutdown
+      end
+
+      should_write_lines "startup", "before", "around 1", "task 1", "around 2", "after", "before", "around 1", "task 2", "around 2", "after", "shutdown"
+    end
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: process_pool
 version: !ruby/object:Gem::Version 
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors: 
 - Adam Pohorecki
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-20 00:00:00 +01:00
+date: 2010-02-18 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -31,18 +31,23 @@ extensions: []
 extra_rdoc_files: 
 - LICENSE
 - README
+- TODO
 files: 
 - .gitignore
 - Gemfile
 - LICENSE
 - README
 - Rakefile
+- TODO
 - VERSION
+- lib/anonymous_extension.rb
+- lib/base_worker_extension.rb
 - lib/init.rb
 - lib/process_pool.rb
 - lib/simple_logger.rb
 - lib/simple_queue.rb
 - process_pool.gemspec
+- test/anonymous_extension_test.rb
 - test/process_pool_test.rb
 - test/simple_queue_test.rb
 - test/test_helper.rb
@@ -75,6 +80,7 @@ signing_key:
 specification_version: 3
 summary: ProcessPool with interchangeable job queue backends for Ruby
 test_files: 
+- test/anonymous_extension_test.rb
 - test/process_pool_test.rb
 - test/simple_queue_test.rb
 - test/test_helper.rb