pigeon 0.3.0 → 0.4.0

data/lib/pigeon.rb

@@ -1,12 +1,20 @@
 module Pigeon
   # == Autoloads ============================================================
   
+  autoload(:Backlog, 'pigeon/backlog')
+  autoload(:Dispatcher, 'pigeon/dispatcher')
   autoload(:Engine, 'pigeon/engine')
   autoload(:Launcher, 'pigeon/launcher')
   autoload(:OptionAccessor, 'pigeon/option_accessor')
   autoload(:Pidfile, 'pigeon/pidfile')
+  autoload(:Processor, 'pigeon/processor')
   autoload(:Queue, 'pigeon/queue')
+  autoload(:Scheduler, 'pigeon/scheduler')
+  autoload(:SortedArray, 'pigeon/sorted_array')
   autoload(:Support, 'pigeon/support')
+  autoload(:Task, 'pigeon/task')
 end
 
-require 'pigeon/logger'
+# NOTE: This file needs to be directly loaded due to some kind of peculiar
+# issue where requiring it later causes a run-time Interrupt exception.
+require 'pigeon/logger'

data/pigeon.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{pigeon}
-  s.version = "0.3.0"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["tadman"]
-  s.date = %q{2010-10-19}
+  s.date = %q{2010-11-09}
   s.default_executable = %q{launcher.example}
   s.description = %q{Pigeon is a simple way to get started building an EventMachine engine that's intended to run as a background job.}
   s.email = %q{github@tadman.ca}
@@ -27,20 +27,31 @@ Gem::Specification.new do |s|
      "VERSION",
      "bin/launcher.example",
      "lib/pigeon.rb",
+     "lib/pigeon/dispatcher.rb",
      "lib/pigeon/engine.rb",
      "lib/pigeon/launcher.rb",
      "lib/pigeon/logger.rb",
      "lib/pigeon/option_accessor.rb",
      "lib/pigeon/pidfile.rb",
+     "lib/pigeon/processor.rb",
      "lib/pigeon/queue.rb",
+     "lib/pigeon/scheduler.rb",
+     "lib/pigeon/sorted_array.rb",
      "lib/pigeon/support.rb",
+     "lib/pigeon/task.rb",
      "pigeon.gemspec",
      "test/helper.rb",
-     "test/test_pigeon.rb",
-     "test/test_pigeon_engine.rb",
-     "test/test_pigeon_launcher.rb",
-     "test/test_pigeon_option_accessor.rb",
-     "test/test_pigeon_queue.rb"
+     "test/unit/pigeon_backlog_test.rb",
+     "test/unit/pigeon_dispatcher_test.rb",
+     "test/unit/pigeon_engine_test.rb",
+     "test/unit/pigeon_launcher_test.rb",
+     "test/unit/pigeon_option_accessor_test.rb",
+     "test/unit/pigeon_processor_test.rb",
+     "test/unit/pigeon_queue_test.rb",
+     "test/unit/pigeon_scheduler_test.rb",
+     "test/unit/pigeon_sorted_array_test.rb",
+     "test/unit/pigeon_task_test.rb",
+     "test/unit/pigeon_test.rb"
   ]
   s.homepage = %q{http://github.com/tadman/pigeon}
   s.rdoc_options = ["--charset=UTF-8"]
@@ -49,11 +60,17 @@ Gem::Specification.new do |s|
   s.summary = %q{Simple daemonized EventMachine engine framework with plug-in support}
   s.test_files = [
     "test/helper.rb",
-     "test/test_pigeon.rb",
-     "test/test_pigeon_engine.rb",
-     "test/test_pigeon_launcher.rb",
-     "test/test_pigeon_option_accessor.rb",
-     "test/test_pigeon_queue.rb"
+     "test/unit/pigeon_backlog_test.rb",
+     "test/unit/pigeon_dispatcher_test.rb",
+     "test/unit/pigeon_engine_test.rb",
+     "test/unit/pigeon_launcher_test.rb",
+     "test/unit/pigeon_option_accessor_test.rb",
+     "test/unit/pigeon_processor_test.rb",
+     "test/unit/pigeon_queue_test.rb",
+     "test/unit/pigeon_scheduler_test.rb",
+     "test/unit/pigeon_sorted_array_test.rb",
+     "test/unit/pigeon_task_test.rb",
+     "test/unit/pigeon_test.rb"
   ]
 
   if s.respond_to? :specification_version then

data/test/helper.rb

@@ -4,6 +4,8 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.expand_path(*%w[ .. lib ]), File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
+require 'timeout'
+
 require 'rubygems'
 
 if (Gem.available?('eventmachine'))
@@ -15,4 +17,21 @@ end
 require 'pigeon'
 
 class Test::Unit::TestCase
+  def assert_timeout(time, message = nil, &block)
+    Timeout::timeout(time, &block)
+  rescue Timeout::Error
+    flunk(message || 'assert_timeout timed out')
+  end
+  
+  def assert_eventually(time = nil, message = nil, &block)
+    start_time = Time.now.to_i
+
+    while (!block.call)
+      select(nil, nil, nil, 0.1)
+      
+      if (time and (Time.now.to_i - start_time > time))
+        flunk(message || 'assert_eventually timed out')
+      end
+    end
+  end
 end

data/test/unit/pigeon_backlog_test.rb

@@ -0,0 +1,25 @@
+require File.expand_path(File.join(*%w[ .. helper ]), File.dirname(__FILE__))
+
+class PigeonQueueTest < Test::Unit::TestCase
+  def test_empty_queue
+    queue = Pigeon::Queue.new
+    
+    assert queue.empty?
+    assert_equal 0, queue.length
+    
+    assert_equal nil, queue.pop
+  end
+  
+  def test_queue_cycling
+    queue = Pigeon::Queue.new
+    
+    task = Pigeon::Task.new
+    
+    queue << task
+    
+    assert_equal 1, queue.length
+    assert !queue.empty?
+    
+    found_task = queue.pop
+  end
+end

data/test/unit/pigeon_dispatcher_test.rb

@@ -0,0 +1,62 @@
+require File.expand_path(File.join(*%w[ .. helper ]), File.dirname(__FILE__))
+
+class PigeonDispatcherTest < Test::Unit::TestCase
+  def test_routine_dispatching
+    dispatcher = Pigeon::Dispatcher.new
+    
+    assert_equal Pigeon::Dispatcher.thread_limit, dispatcher.thread_limit
+    
+    checks = { }
+    
+    count = 100
+    
+    count.times do |n|
+      dispatcher.perform do
+        x = 0
+
+        1000.times do
+          dispatcher.perform do 
+            x += 1
+          end
+        end
+        
+        checks[n] = x
+      end
+    end
+    
+    assert_timeout(10) do
+      dispatcher.wait!
+    end
+
+    assert_equal 0, dispatcher.backlog_size
+    assert_equal 0, dispatcher.thread_count
+    assert dispatcher.empty?
+    assert_equal [ ], dispatcher.exceptions
+    assert !dispatcher.exceptions?
+    
+    assert_equal (0..count - 1).to_a, checks.keys.sort
+  end
+
+  def test_dispatch_variants
+    dispatcher = Pigeon::Dispatcher.new
+    
+    result = [ ]
+    
+    dispatcher.perform do
+      result[0] = :a
+    end
+    
+    dispatcher.perform(:b) do |b|
+      result[1] = b
+    end
+
+    dispatcher.perform(:c, :d) do |c, d|
+      result[2] = c
+      result[3] = d
+    end
+    
+    dispatcher.wait!
+    
+    assert_equal [ :a, :b, :c, :d ], result
+  end
+end

data/test/{test_pigeon_engine.rb → unit/pigeon_engine_test.rb}

@@ -1,4 +1,4 @@
-require File.expand_path(File.join(*%w[ helper ]), File.dirname(__FILE__))
+require File.expand_path(File.join(*%w[ .. helper ]), File.dirname(__FILE__))
 
 module TestModule
   def self.included(engine)

data/test/{test_pigeon_launcher.rb → unit/pigeon_launcher_test.rb}

@@ -1,4 +1,4 @@
-require File.expand_path(File.join(*%w[ helper ]), File.dirname(__FILE__))
+require File.expand_path(File.join(*%w[ .. helper ]), File.dirname(__FILE__))
 
 class PigeonLauncherTest < Test::Unit::TestCase
   def test_default_launcher

data/test/{test_pigeon_option_accessor.rb → unit/pigeon_option_accessor_test.rb}

@@ -1,4 +1,4 @@
-require File.expand_path(File.join(*%w[ helper ]), File.dirname(__FILE__))
+require File.expand_path(File.join(*%w[ .. helper ]), File.dirname(__FILE__))
 
 class OptionClass
   extend Pigeon::OptionAccessor

data/test/unit/pigeon_processor_test.rb

@@ -0,0 +1,94 @@
+require File.expand_path(File.join(*%w[ .. helper ]), File.dirname(__FILE__))
+
+class PigeonProcessorTest < Test::Unit::TestCase
+  class TaggedTask < Pigeon::Task
+    attr_accessor :tag
+    
+    def initialize(engine, tag)
+      super(engine)
+      @tag = tag
+    end
+    
+    def inspect
+      "<#{@tag}>"
+    end
+  end
+
+  def engine
+    @engine ||= Pigeon::Engine.new
+  end
+
+  def test_empty_processor
+    queue = Pigeon::Queue.new
+    
+    processor = Pigeon::Processor.new(queue)
+    
+    assert_equal false, processor.task?
+    
+    assert_equal true, processor.accept?(Pigeon::Task.new(engine))
+    
+    assert processor.id
+  end
+  
+  def test_simple_filter
+    queue = Pigeon::Queue.new
+    
+    processor = Pigeon::Processor.new(queue) do |task|
+      (task.tag % 2) == 1
+    end
+    
+    assert_equal false, processor.task?
+    
+    queue << TaggedTask.new(engine, 0)
+    
+    assert_equal false, processor.task?
+    assert_equal 1, queue.length
+
+    queue << TaggedTask.new(engine, 1)
+    
+    assert_equal true, processor.task?
+    assert_equal 1, queue.length
+    
+    assert_eventually(5) do
+      !processor.task?
+    end
+    
+    assert_equal 1, queue.length
+  end
+
+  def test_on_backlog
+    queue = Pigeon::Queue.new
+    
+    100.times do |n|
+      queue << TaggedTask.new(engine, n)
+    end
+    
+    processor = Pigeon::Processor.new(queue)
+    
+    assert_eventually(5) do
+      queue.empty?
+    end
+  end
+
+  def test_multiple_processors
+    queue = Pigeon::Queue.new
+    count = 10000
+    
+    count.times do |n|
+      queue << TaggedTask.new(engine, n)
+    end
+    
+    assert_equal count, queue.length
+    
+    processors = (0..9).to_a.collect do
+      Pigeon::Processor.new(queue)
+    end
+    
+    assert_eventually(10) do
+      queue.empty?
+    end
+    
+    assert_equal 0, processors.select(&:task?).length
+    assert_equal 0, queue.length
+  end
+end

data/test/unit/pigeon_queue_test.rb

@@ -0,0 +1,197 @@
+require File.expand_path(File.join(*%w[ .. helper ]), File.dirname(__FILE__))
+
+class PigeonQueueTest < Test::Unit::TestCase
+  class TaggedTask < Pigeon::Task
+    attr_accessor :tag
+    
+    def initialize(engine, tag)
+      super(engine)
+      @tag = tag
+    end
+    
+    def inspect
+      "<#{@tag}>"
+    end
+  end
+  
+  def engine
+    @engine ||= Pigeon::Engine.new
+  end
+
+  def test_empty_state
+    queue = Pigeon::Queue.new
+    
+    assert_equal 0, queue.length
+    assert_equal true, queue.empty?
+    
+    assert_equal nil, queue.pop
+  end
+  
+  def test_cycling
+    queue = Pigeon::Queue.new
+    
+    task = Pigeon::Task.new(engine)
+    
+    assert_equal task, queue << task
+    
+    assert queue.peek
+    assert_equal 1, queue.length
+    assert !queue.empty?
+    
+    found_task = queue.pop
+    
+    assert_equal task, found_task
+    
+    assert_equal 0, queue.length
+    assert queue.empty?
+  end
+    
+  def test_filtering
+    queue = Pigeon::Queue.new
+
+    tasks = (0..9).to_a.collect do |n|
+      queue << TaggedTask.new(engine, n)
+    end
+    
+    assert_equal (0..9).to_a, tasks.to_a.collect(&:tag)
+    
+    assert_equal tasks[0], queue.peek
+
+    selected_task = queue.peek do |task|
+      task.tag > 0
+    end
+
+    assert_equal tasks[1], selected_task
+    
+    queue.filter(:over_7) do |task|
+      task.tag > 7
+    end
+
+    assert_equal tasks[8], queue.peek(:over_7)
+    assert_equal 2, queue.length(:over_7)
+    
+    pulled_task = queue.pop(:over_7)
+    
+    assert_equal 9, queue.length
+
+    assert_equal tasks[9], queue.peek(:over_7)
+    assert_equal 1, queue.length(:over_7)
+    
+    queue.pop(:over_7)
+
+    assert_equal nil, queue.peek(:over_7)
+    assert_equal 0, queue.length(:over_7)
+    assert_equal true, queue.empty?(:over_7)
+    
+    new_task = queue << TaggedTask.new(engine, 10)
+    
+    assert_equal new_task, queue.peek(:over_7)
+    assert_equal 1, queue.length(:over_7)
+    assert_equal false, queue.empty?(:over_7)
+    
+    queue.claim(new_task)
+    
+    assert_equal nil, queue.peek(:over_7)
+    assert_equal 0, queue.length(:over_7)
+    assert_equal true, queue.empty?(:over_7)
+  end
+
+  def test_observe
+    queue = Pigeon::Queue.new
+    
+    tasks = (0..9).to_a.collect do |n|
+      queue << TaggedTask.new(engine, n)
+    end
+    
+    queue.filter(:odd) do |task|
+      task.tag % 2 == 1
+    end
+    
+    assert_equal tasks[1], queue.peek(:odd)
+    assert_equal 5, queue.length(:odd)
+    
+    assert_equal [ tasks[1], tasks[3], tasks[5], tasks[7], tasks[9] ], queue.pull(:odd)
+    
+    assert_equal 5, queue.length
+    assert_equal 0, queue.length(:odd)
+    
+    added_odd = nil
+    
+    queue.observe(:odd) do |task|
+      added_odd = task
+    end
+    
+    queue << TaggedTask.new(engine, 10)
+    
+    assert_equal nil, added_odd
+
+    odd_1 = queue << TaggedTask.new(engine, 11)
+    
+    assert_equal odd_1, added_odd
+
+    claimed_task = nil
+    has_run = false
+    
+    queue.observe(:odd) do |task|
+      claimed_task = queue.claim(task)
+      has_run = true
+    end
+    
+    # Observer callbacks are not triggered on existing data, only on new
+    # insertions.
+    assert_equal false, has_run
+    assert_equal nil, claimed_task
+    assert_equal 7, queue.length
+    assert_equal 1, queue.length(:odd)
+
+    queue << TaggedTask.new(engine, 12)
+    
+    assert_equal nil, claimed_task
+    assert_equal 8, queue.length
+    assert_equal 1, queue.length(:odd)
+
+    odd_2 = queue << TaggedTask.new(engine, 13)
+    
+    # Adding a task that matches the filter triggers the callback.
+    assert_equal odd_2, claimed_task
+    assert_equal true, has_run
+    
+    # Clear out all of the odd entries.
+    queue.pull(:odd)
+    
+    claimed_task = nil
+    has_run = false
+
+    queue << TaggedTask.new(engine, 14)
+
+    assert_equal nil, claimed_task
+    assert_equal false, has_run
+
+    odd_2 = queue << TaggedTask.new(engine, 15)
+    
+    assert_equal odd_2, claimed_task
+    assert_equal true, has_run
+
+    assert_equal 8, queue.length
+    assert_equal 0, queue.length(:odd)
+  end
+  
+  def test_can_add_during_observe
+    queue = Pigeon::Queue.new
+    
+    queue.observe do |task|
+      if (task.tag < 10)
+        queue.claim(task)
+
+        queue << TaggedTask.new(engine, task.tag + 1)
+      end
+    end
+    
+    queue << TaggedTask.new(engine, 0)
+
+    assert queue.peek
+    assert_equal 10, queue.peek.tag
+    assert_equal 1, queue.length
+    assert queue.peek
+  end
+end

data/test/unit/pigeon_scheduler_test.rb

@@ -0,0 +1,70 @@
+require File.expand_path(File.join(*%w[ .. helper ]), File.dirname(__FILE__))
+
+class PigeonSchedulerTest < Test::Unit::TestCase
+  class TaggedTask < Pigeon::Task
+    attr_accessor :tag
+    
+    def initialize(engine, tag)
+      super(engine)
+      @tag = tag
+    end
+    
+    def inspect
+      "<#{@tag}>"
+    end
+  end
+
+  def engine
+    @engine ||= Pigeon::Engine.new
+  end
+  
+  def test_defaults
+    scheduler = Pigeon::Scheduler.new
+    
+    assert_equal true, scheduler.default_queue.empty?
+  end
+  
+  def test_queued
+    queue = Pigeon::Queue.new
+    
+    scheduler = Pigeon::Scheduler.new(queue)
+
+    count = 1000
+    
+    count.times do |n|
+      queue << TaggedTask.new(engine, n)
+    end
+    
+    assert_eventually(5) do
+      queue.empty?
+    end
+    
+    assert_equal 0, scheduler.processors.select(&:task?).length
+    assert_equal 0, queue.length
+  end
+
+  def test_add
+    queue = Pigeon::Queue.new
+    
+    scheduler = Pigeon::Scheduler.new(queue)
+
+    count = 1000
+    
+    backlog = [ ]
+    count.times do |n|
+      scheduler.add(TaggedTask.new(engine, n * 2 + 1))
+      backlog << TaggedTask.new(engine, n * 2)
+    end
+    
+    scheduler.add(backlog)
+    
+    assert !queue.empty?
+    
+    assert_eventually(5) do
+      queue.empty?
+    end
+    
+    assert_equal 0, scheduler.processors.select(&:task?).length
+    assert_equal 0, queue.length
+  end
+end

data/test/unit/pigeon_sorted_array_test.rb

@@ -0,0 +1,52 @@
+require File.expand_path(File.join(*%w[ .. helper ]), File.dirname(__FILE__))
+
+class PigeonSortedArrayTest < Test::Unit::TestCase
+  def test_empty_state
+    array = Pigeon::SortedArray.new
+    
+    assert array.empty?
+    assert array.is_a?(Array)
+    assert_equal [ ], array
+    
+    dup = array.dup
+    
+    assert_equal Pigeon::SortedArray, dup.class
+    
+    combined = (array + array)
+    
+    assert_equal [ ], combined
+    assert_equal Pigeon::SortedArray, combined.class
+  end
+  
+  def test_does_sorting
+    array = Pigeon::SortedArray.new
+    
+    test_array = (0..100).to_a.reverse
+    
+    array += test_array
+    
+    assert_equal (0..100).to_a, array
+  end
+
+  def test_does_sorting_with_insertion_in_order
+    array = Pigeon::SortedArray.new
+
+    10.times do |n|
+      array << n.to_f
+    end
+    
+    assert_equal (0..9).to_a.collect(&:to_f), array
+  end
+
+  def test_does_sorting_with_insertion_random_order
+    array = Pigeon::SortedArray.new
+    
+    srand
+    (0..10000).to_a.sort_by { rand }.each do |i|
+      array << i
+    end
+  
+    assert_equal (0..10000).to_a, array
+    assert_equal 10001, array.length
+  end
+end