pigeon 0.3.0 → 0.4.0

data/README.rdoc

@@ -7,14 +7,12 @@ batch processing, or for providing specific network services.
 Installation should be as simple as:
 
     gem install pigeon
-    
-Your first Pigeon engine can be defined quite simply:
+
+Your first Pigeon engine can be defined by declaring a subclass:
 
     class MyEngine < Pigeon::Engine
-      def self.included(engine)
-        engine.after_start do
-          # Operations to be performed after start
-        end
+      after_start do
+        # Operations to be performed after start
       end
     end
     
@@ -30,10 +28,11 @@ A primary function of an engine might be to intermittently perform a task.
 Several methods exist to facilitate this:
 
     class MyEngine < Pigeon::Engine
-      def self.included(engine)
-        engine.after_start do
-          engine.periodically_trigger_task(10) do
-          end
+      after_start do
+        periodically_trigger_task(10) do
+          # Arbitrary block of code is executed every ten seconds but only
+          # one instance of this block can be running at a time.
+          do_stuff_every_ten_seconds
         end
       end
     end
@@ -43,60 +42,37 @@ Starting your application can be done with a wrapper script that is constructed
 An example would look like:
 
     #!/usr/bin/env ruby
+    
+    require 'rubygems'
+    gem 'pigeon'
 
-    COMMAND_NAME = 'launcher'
-    engine = engine
+    # Adjust search path to include the ../lib directory
+    $LOAD_PATH << File.expand_path(
+      File.join(*%w[ .. lib ]), File.dirname(__FILE__)
+    )
 
-    options = {
-      :dir => engine.pid_dir,
-      :debug => true, # ((ENV['RAILS_ENV'] == 'production') ? ENV['PINGITY_DEBUG'] : true),
-      :modules => [ ]
-    }
+    # Use Pigeon::Launcher to launch your own engine by replacing
+    # the parameter Pigeon::Engine with your specific subclass.
+    Pigeon::Launcher.new(Pigeon::Engine).handle_args(ARGV)
+    
+== Components
 
-    begin
-      case (command)
-      when 'start'
-        engine.start(options) do |pid|
-          puts "Pigeon Engine now running. [%d]" % pid
-        end
-      when 'stop'
-        engine.stop(options) do |pid|
-          if (pid)
-            puts "Pigeon Engine shut down. [%d]" % pid
-          else
-            puts "Pigeon Engine was not running."
-          end
-        end
-      when 'restart'
-        engine.restart(options) do |old_pid, new_pid|
-          if (old_pid)
-            puts "Pigeon Engine terminated. [%d]" % old_pid
-          end
-          puts "Pigeon Engine now running. [%d]" % new_pid
-        end
-      when 'status'
-        engine.status(options) do |pid|
-          if (pid)
-            puts "Pigeon Engine running. [%d]" % pid
-          else
-            puts "Pigeon Engine is not running."
-          end
-        end
-      when 'run'
-        options[:logger] = Pigeon::Logger.new(STDOUT)
+There are several key components used by Pigeon to create an event-driven
+engine.
+
+=== Pigeon::Dispatcher
+
+The dispatcher functions as a thread pool for processing small, discrete
+operations. These threads are created on demand and destroyed when no longer
+in use. By limiting the number of threads a pool can contain it is possible
+to schedule sequential operations, manage control over a single shared
+resource, or to run through large lists of operations in parallel.
+
+=== Pigeon::SortedArray
+
+This utility class provides a simple self-sorting array. This is used as a
+priority queue within the Pigeon::Queue.
 
-        engine.run(options) do |pid|
-          puts "Pigeon Engine now running. [%d]" % pid
-          puts "Use ^C to terminate."
-        end
-      else
-        puts "Usage: #{COMMAND_NAME} [start|stop|restart|status|run]"
-      end
-    rescue Interrupt
-      puts "Shutting down."
-      exit(0)
-    end
-    
 == Status
 
 This engine is currently in development.

data/Rakefile

@@ -20,23 +20,10 @@ end
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
+  test.pattern = 'test/**/*_test.rb'
   test.verbose = true
 end
 
-begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |test|
-    test.libs << 'test'
-    test.pattern = 'test/**/test_*.rb'
-    test.verbose = true
-  end
-rescue LoadError
-  task :rcov do
-    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-  end
-end
-
 task :test => :check_dependencies
 
 task :default => :test

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

data/bin/launcher.example

@@ -1,4 +1,19 @@
 #!/usr/bin/env ruby
+#
+# launcher    Pigeon launcher
+#
+# chkconfig: - 90 10
+# description: This is an example of a Pigeon launcher.
+# processname: launcher
+# pidfile: /var/run/pigeon.pid
+#
+### BEGIN INIT INFO
+# Provides: pigeon
+# Required-Start: $local_fs $remote_fs $network $named
+# Required-Stop: $local_fs $remote_fs $network
+# Short-Description: start and stop the Pigeon engine
+# Description: This is an example of a Pigeon launcher.
+### END INIT INFO
 
 $LOAD_PATH << File.expand_path(File.join(*%w[ .. lib ]), File.dirname(__FILE__))
 require 'pigeon'

data/lib/pigeon/dispatcher.rb

@@ -0,0 +1,93 @@
+require 'thwait'
+
+class Pigeon::Dispatcher
+  # == Extensions ===========================================================
+  
+  extend Pigeon::OptionAccessor
+
+  # == Constants ============================================================
+
+  # == Properties ===========================================================
+  
+  option_accessor :thread_limit,
+    :default => 24
+
+  attr_reader :exceptions
+
+  # == Class Methods ========================================================
+
+  # == Instance Methods =====================================================
+  
+  # Creates a new instance of a dispatcher. An optional limit parameter is
+  # used to specify how many threads can be used, which if nil will use the
+  # concurrency_limit established for the class.
+  def initialize(limit = nil)
+    @thread_limit = limit
+    @backlog = [ ]
+    @threads = [ ]
+    @exceptions = [ ]
+    @sempaphore = Mutex.new
+  end
+  
+  def perform(*args, &block)
+    @backlog << [ block, args, caller(0) ]
+
+    @sempaphore.synchronize do
+      if (@threads.length < self.thread_limit and @threads.length < @backlog.length)
+        create_thread
+      end
+    end
+  end
+  
+  # Returns true if there are no operations in the backlog queue or running,
+  # false otherwise.
+  def empty?
+    @backlog.empty? and @threads.empty?
+  end
+  
+  # Returns true if any exceptions have been generated, false otherwise.
+  def exceptions?
+    !@exceptions.empty?
+  end
+
+  # Returns the number of items in the backlog queue.
+  def backlog_size
+    @backlog.length
+  end
+  
+  # Returns the current number of threads executing.
+  def thread_count
+    @threads.length
+  end
+  
+  # Waits until all operations have completed, including the backlog.
+  def wait!
+    while (!@threads.empty?)
+      ThreadsWait.new(@threads).join
+    end
+  end
+  
+protected
+  def create_thread
+    @threads << Thread.new do
+      Thread.current.abort_on_exception = true
+    
+      begin
+        while (block = @backlog.pop)
+          begin
+            block[0].call(*block[1])
+          rescue Object => e
+            puts "#{e.class}: #{e} #{e.backtrace.join("\n")}"
+            @exceptions << e
+          end
+        
+          Thread.pass
+        end
+      ensure
+        @sempaphore.synchronize do
+          @threads.delete(Thread.current)
+        end
+      end
+    end
+  end
+end

data/lib/pigeon/engine.rb

@@ -1,6 +1,5 @@
 require 'eventmachine'
 require 'socket'
-require 'digest/sha1'
 
 class Pigeon::Engine
   # == Submodules ===========================================================
@@ -18,33 +17,30 @@ class Pigeon::Engine
   # == Properties ===========================================================
   
   option_accessor :logger
-
   option_accessor :name
   option_accessor :pid_file_name
   option_accessor :foreground,
     :boolean => true
   option_accessor :debug,
     :boolean => true
-    
   option_accessor :engine_log_name,
     :default => 'engine.log'
   option_accessor :engine_logger
-
   option_accessor :query_log_name,
     :default => 'query.log'
   option_accessor :query_logger
-  
   option_accessor :try_pid_dirs,
     :default => %w[
       /var/run
       /tmp
     ].freeze
-
   option_accessor :try_log_dirs,
     :default => %w[
       /var/log
       /tmp
     ].freeze
+    
+  attr_reader :id
 
   # == Constants ============================================================
   
@@ -89,6 +85,8 @@ class Pigeon::Engine
   
   # Launches the engine with the specified options
   def self.launch(options = nil)
+    engine = nil
+    
     EventMachine.run do
       engine = new(options)
       
@@ -98,8 +96,12 @@ class Pigeon::Engine
         engine.terminate
       end
 
+      Pigeon::Engine.register_engine(engine)
+
       engine.run
     end
+
+    Pigeon::Engine.unregister_engine(engine)
   end
   
   def self.pid_file
@@ -181,10 +183,30 @@ class Pigeon::Engine
       Pigeon::Logger.new(f)
     end
   end
+  
+  # Returns a handle to the engine currently running, or nil if no engine is
+  # currently active.
+  def self.default_engine
+    @engines and @engines[0]
+  end
+
+  # Registers the engine as running. The first engine running will show up
+  # as the default engine.
+  def self.register_engine(engine)
+    @engines ||= [ ]
+    @engines << engine
+  end
+  
+  # Removes the engine from the list of running engines.
+  def self.unregister_engine(engine)
+    @engines.delete(engine)
+  end
 
   # == Instance Methods =====================================================
 
   def initialize(options = nil)
+    @id = Pigeon::Support.unique_id
+
     @options = options || { }
     
     @task_lock = Mutex.new
@@ -193,7 +215,7 @@ class Pigeon::Engine
     self.logger ||= self.engine_logger
     self.logger.level = Pigeon::Logger::DEBUG if (self.debug?)
     
-    @queue = { }
+    @dispatcher = { }
     
     run_chain(:after_initialize)
   end
@@ -203,19 +225,6 @@ class Pigeon::Engine
     Socket.gethostname
   end
 
-  # Returns a unique 160-bit identifier for this engine expressed as a 40
-  # character hexadecimal string. The first 32-bit sequence is a timestamp
-  # so these numbers increase over time and can be used to identify when
-  # a particular instance was launched.
-  def id
-    @id ||= '%8x%s' % [
-      Time.now.to_i,
-      Digest::SHA1.hexdigest(
-        '%.8f%8x' % [ Time.now.to_f, rand(1 << 32) ]
-      )[0, 32]
-    ]
-  end
-
   # Handles the run phase of the engine, triggers the before_start and
   # after_start events accordingly.
   def run
@@ -256,7 +265,7 @@ class Pigeon::Engine
     
     return if (@task_locks[task_name].locked?)
     
-    @task_lock[task_name].synchronize do
+    @task_locks[task_name].synchronize do
       yield if (block_given?)
     end
   end
@@ -273,7 +282,7 @@ class Pigeon::Engine
   
   # Used to defer a block of work for near-immediate execution. Is a 
   # wrapper around EventMachine#defer and does not perform as well as using
-  # the alternate queue method.
+  # the alternate dispatch method.
   def defer(&block)
     EventMachine.defer(&block)
   end
@@ -288,12 +297,12 @@ class Pigeon::Engine
     run_chain(:after_stop)
   end
   
-  # Used to queue a block for immediate processing on a background thread.
+  # Used to dispatch a block for immediate processing on a background thread.
   # An optional queue name can be used to sequence tasks properly. The main
   # queue has a large number of threads, while the named queues default
   # to only one so they can be processed sequentially.
-  def queue(name = :default, &block)
-    target_queue = @queue[name] ||= Pigeon::Queue.new(name == :default ? nil : 1)
+  def dispatch(name = :default, &block)
+    target_queue = @dispatcher[name] ||= Pigeon::Dispatcher.new(name == :default ? nil : 1)
     
     target_queue.perform(&block)
   end
@@ -328,12 +337,12 @@ class Pigeon::Engine
 
   # Returns true if the debug option was set, false otherwise.
   def debug?
-    !!@options[:debug]
+    !!self.debug
   end
   
   # Returns true if running in the foreground, false otherwise.
   def foreground?
-    !!@options[:foreground]
+    !!self.foreground
   end
 
 protected

data/lib/pigeon/option_accessor.rb

@@ -16,6 +16,8 @@ module Pigeon::OptionAccessor
   # but these defaults can be over-ridden in subclasses and instances
   # without interference. Optional hash at end of list can be used to set:
   #  * :default => Assigns a default value which is otherwise nil
+  #  * :boolean => If true, creates an additional name? method and will
+  #                convert all assigned values to a boolean true/false.
   def option_reader(*names)
     names = [ names ].flatten.compact
     options = names.last.is_a?(Hash) ? names.pop : { }

data/lib/pigeon/processor.rb

@@ -0,0 +1,66 @@
+class Pigeon::Processor
+  # == Exceptions ===========================================================
+  
+  class AlreadyBoundToQueue < Exception
+  end
+  
+  # == Constants ============================================================
+
+  # == Properties ===========================================================
+  
+  attr_reader :task
+  attr_reader :id
+
+  # == Class Methods ========================================================
+
+  # == Instance Methods =====================================================
+  
+  def initialize(queue = nil, &filter)
+    @id = Pigeon::Support.unique_id
+    @lock = Mutex.new
+    @filter = filter || lambda { |task| true }
+    
+    self.queue = queue if (queue)
+    
+    switch_to_next_task!
+  end
+  
+  def queue=(queue)
+    raise AlreadyBoundToQueue, @queue if (@queue)
+    
+    @queue = queue
+
+    @queue.observe do |task|
+      @lock.synchronize do
+        if (!@task and @filter.call(task))
+          @task = queue.claim(task)
+      
+          @task.run! do
+            switch_to_next_task!
+          end
+        end
+      end
+    end
+  end
+  
+  def accept?(task)
+    @filter.call(task)
+  end
+  
+  def task?
+    !!@task
+  end
+  
+protected
+  def switch_to_next_task!
+    @lock.synchronize do
+      @task = nil
+
+      if (@task = @queue.pop(&@filter))
+        @task.run! do
+          switch_to_next_task!
+        end
+      end
+    end
+  end
+end