pigeon 0.3.0 → 0.4.0

data/lib/pigeon/queue.rb

@@ -1,66 +1,299 @@
 class Pigeon::Queue
   # == Constants ============================================================
+  
+  # == Exceptions ===========================================================
 
-  DEFAULT_CONCURRENCY_LIMIT = 24
+  class BlockRequired < Exception
+  end
+  
+  class TaskNotQueued < Exception
+    def initialize(task = nil)
+      @task = task
+    end
+    
+    def inspect
+      "Task #{@task.inspect} not queued."
+    end
+    alias_method :to_s, :inspect
+  end
 
-  # == Properties ===========================================================
+  # == Extensions ===========================================================
 
-  attr_reader :exceptions
+  # == Relationships ========================================================
+
+  # == Scopes ===============================================================
+
+  # == Callbacks ============================================================
+
+  # == Validations ==========================================================
 
   # == Class Methods ========================================================
+  
+  def self.filters
+    @filters ||= {
+      nil => lambda { |task| true }
+    }
+  end
+  
+  def self.filter(name, &block)
+    filters[name] = block
+  end
 
   # == Instance Methods =====================================================
-  
-  def initialize(limit = nil)
-    @limit = limit || DEFAULT_CONCURRENCY_LIMIT
-    @blocks = [ ]
-    @threads = [ ]
-    @exceptions = [ ]
+
+  def initialize(&block)
+    @filter_lock = Mutex.new
+    @observer_lock = Mutex.new
+
+    @claimable_task = { }
+    @filters = self.class.filters.dup
+    @observers = { }
+    @next_task = { }
+    @insert_backlog = [ ]
+    
+    if (block_given?)
+      @sort_by = block
+    else
+      @sort_by = lambda { |a,b| a.priority <=> b.priority }
+    end
+
+    @tasks = Pigeon::SortedArray.new(&@sort_by)
   end
   
-  def perform(*args, &block)
-    @blocks << [ block, args, caller(0) ]
+  def sort_by(&block)
+    raise BlockRequired unless (block_given?)
 
-    if (@threads.length < @limit and @threads.length < @blocks.length)
-      create_thread
+    @sort_by = block
+    @filter_lock.synchronize do
+      @tasks = Pigeon::SortedArray.new(&@sort_by) + @tasks
+      
+      @next_task = { }
     end
   end
   
-  def empty?
-    @blocks.empty? and @threads.empty?
+  def observe(filter_name = nil, &block)
+    raise BlockRequired unless (block_given?)
+    
+    @observer_lock.synchronize do
+      @observers[filter_name] ||= [ ]
+    end
+
+    @observers[filter_name] << block
+    
+    task = assign_next_task(filter_name)
   end
   
-  def exceptions?
-    !@exceptions.empty?
+  def filter(filter_name, &block)
+    raise BlockRequired unless (block_given?)
+
+    @filter_lock.synchronize do
+      @filters[filter_name] = block
+    end
+    
+    assign_next_task(filter_name)
   end
+  
+  def <<(task)
+    # If there is an insert operation already in progress, put this task in
+    # the backlog for subsequent processing.
+    
+    if (@observer_lock.locked?)
+      @insert_backlog << task
+      return task
+    end
+    
+    active_task = task
+    
+    while (active_task) do
+      # Set the claimable task flag for this task since it is not yet in the
+      # actual task queue.
+      @filter_lock.synchronize do
+        @claimable_task[active_task] = true
+      end
+    
+      unless (@observers.empty?)
+        @observer_lock.synchronize do
+          @observers.each do |filter_name, list|
+            # Check if this task matches the filter restrictions, and if it
+            # does then call the observer chain in order.
+            if (@filters[filter_name].call(active_task))
+              @observers[filter_name].each do |proc|
+                case (proc.arity)
+                when 2
+                  proc.call(self, active_task)
+                else
+                  proc.call(active_task)
+                end
+
+                # An observer callback has the opportunity to claim a task,
+                # and if it does, the claimable task flag will be false. Loop
+                # only while the task is claimable.
+                break unless (@claimable_task[active_task])
+              end
+            end
+          end
+        end
+      end
+
+        # If this task wasn't claimed by an observer then insert it in the
+        # main task queue.
+      if (@claimable_task.delete(active_task))
+        @filter_lock.synchronize do
+          @tasks << active_task
+          
+          # Update the next task slots for all of the unassigned filters and
+          # trigger observer callbacks as required.
+          @next_task.each do |filter_name, next_task|
+            next if (next_task)
+            
+            if (@filters[filter_name].call(active_task))
+              @next_task[filter_name] = active_task
+            end
+          end
+        end
+      end
+        
+      active_task = @insert_backlog.shift
+    end
 
-  def length
-    @blocks.length
+    task
   end
   
-  def threads
-    @threads.length
+  def each
+    @filter_lock.synchronize do
+      tasks = @tasks.dup
+    end
+    
+    tasks.each do
+      yield(task)
+    end
   end
   
-protected
-  def create_thread
-    @threads << Thread.new do
-      Thread.current.abort_on_exception = true
+  def peek(filter_name = nil, &block)
+    if (block_given?)
+      @filter_lock.synchronize do
+        @tasks.find(&block)
+      end
+    else
+      @next_task[filter_name] ||= begin
+        @filter_lock.synchronize do
+          filter_proc = @filters[filter_name]
       
-      begin
-        while (block = @blocks.pop)
-          begin
-            block[0].call(*block[1])
-          rescue Object => e
-            puts "#{e.class}: #{e} #{e.backtrace.join("\n")}"
-            @exceptions << e
+          filter_proc and @tasks.find(&filter_proc)
+        end
+      end
+    end
+  end
+  
+  def pull(filter_name = nil, &block)
+    unless (block_given?)
+      block = @filters[filter_name]
+    end
+    
+    @filter_lock.synchronize do
+      tasks = @tasks.select(&block)
+      
+      @tasks -= tasks
+      
+      @next_task.each do |filter_name, next_task|
+        if (tasks.include?(@next_task[filter_name]))
+          @next_task[filter_name] = nil
+        end
+      end
+      
+      tasks
+    end
+  end
+
+  def pop(filter_name = nil, &block)
+    @filter_lock.synchronize do
+      task =
+        if (block_given?)
+          @tasks.find(&block)
+        else
+          @next_task[filter_name] || begin
+            filter_proc = @filters[filter_name]
+
+            filter_proc and @tasks.find(&filter_proc)
+          end
+        end
+    
+      if (task)
+        @tasks.delete(task)
+
+        @next_task.each do |filter_name, next_task|
+          if (task == next_task)
+            @next_task[filter_name] = nil
+          end
+        end
+      end
+
+      task
+    end
+  end
+  
+  def claim(task)
+    @filter_lock.synchronize do
+      if (@claimable_task[task])
+        @claimable_task[task] = false
+      elsif (@tasks.delete(task))
+        @next_task.each do |filter_name, next_task|
+          if (task == next_task)
+            @next_task[filter_name] = nil
           end
-          
-          Thread.pass
         end
-      ensure
-        @threads.delete(Thread.current)
+      else
+        raise TaskNotQueued, task
       end
     end
+      
+    task
+  end
+
+  def exist?(task)
+    @filter_lock.synchronize do
+      @tasks.exist?(task)
+    end
+  end
+  
+  def empty?(filter_name = nil, &block)
+    if (block_given?)
+      @filter_lock.synchronize do
+        !@tasks.find(&block)
+      end
+    else
+      !peek(filter_name)
+    end
+  end
+  
+  def length(filter_name = nil, &block)
+    filter_proc = @filters[filter_name] 
+  
+    @filter_lock.synchronize do
+      filter_proc ? @tasks.count(&filter_proc) : nil
+    end
+  end
+  alias_method :size, :length
+  alias_method :count, :length
+  
+  def to_a
+    @filter_lock.synchronize do
+      @tasks.dup
+    end
+  end
+  
+protected
+  def assign_next_task(filter_name)
+    filter = @filters[filter_name]
+
+    return unless (filter)
+    
+    if (task = @next_task[filter_name])
+      return task
+    end
+    
+    @filter_lock.synchronize do
+      @next_task[filter_name] ||= @tasks.find(&filter)
+    end
   end
 end

data/lib/pigeon/scheduler.rb

@@ -0,0 +1,101 @@
+class Pigeon::Scheduler
+  # == Properties ===========================================================
+  
+  attr_reader :queues
+  attr_reader :processors
+
+  # == Class Methods ========================================================
+
+  # == Instance Methods =====================================================
+  
+  # Creates a new scheduler. If queue is specified, then that queue will
+  # become the default queue. One or more processors can be supplied to work
+  # with this queue, though they should be supplied bound to the queue in
+  # order to properly receive tasks.
+  def initialize(queue = nil, *processors)
+    @queues = {
+      nil => queue || Pigeon::Queue.new
+    }
+    
+    processors.flatten!
+    
+    @processors = processors.empty? ? [ Pigeon::Processor.new(@queues[nil]) ] : processors
+  end
+  
+  # Adds one or more tasks to the schedule, where the tasks can be provided
+  # individually, as a list, or as an array.
+  def add(*tasks)
+    tasks.flatten.each do |task|
+      enqueue_task(task)
+    end
+  end
+  
+  # Returns the default queue used for scheduling.
+  def default_queue
+    @queues[nil]
+  end
+
+  # Used to assign the default queue.
+  def default_queue=(queue)
+    @queues[nil] = queue
+  end
+  
+  # Returns the queue with the given name if one is defined, nil otherwise.
+  def queue(queue_name)
+    @queues[queue_name]
+  end
+
+  # Sets the scheduler running.
+  def run!
+    @state = :running
+  end
+
+  # Pauses the scheduler which will prevent additional tasks from being
+  # initiated. Any tasks in progress will continue to run. Tasks can still
+  # be added but will not be executed until the scheduler is running.
+  def pause!
+    @state = :paused
+  end
+  
+  # Stops the scheduler and clears out the queue. No new tasks will be
+  # accepted until the scheduler is in a paused or running state.
+  def stop!
+    @state = :stopped
+  end
+  
+  # Returns true if the scheduler is running, false otherwise.
+  def running?
+    @state == :running
+  end
+  
+  # Returns true if the scheduler is paused, false otherwise.
+  def paused?
+    @state == :paused
+  end
+  
+  # Returns true if the scheduler is stopped, false otherwise.
+  def stopped?
+    @state == :stopped
+  end
+  
+  # Returns the number of tasks that have been queued up.
+  def queue_length
+    @queues.inject(0) do |length, (name, queue)|
+      length + queue.length
+    end
+  end
+  alias_method :queue_size, :queue_length
+
+  # Returns the number of processors that are attached to this scheduler.
+  def processors_count
+    @processors.length
+  end
+
+protected
+  # This method defines how to handle adding a task to the scheduler, which
+  # in this case simply puts it into the default queue. Subclasses should
+  # redefine this to organize tasks as required.
+  def enqueue_task(task)
+    default_queue << task
+  end
+end

data/lib/pigeon/sorted_array.rb

@@ -0,0 +1,56 @@
+class Pigeon::SortedArray < Array
+  # == Exceptions ===========================================================
+  
+  class SortArgumentRequired < Exception
+  end
+  
+  # == Class Methods ========================================================
+
+  # == Instance Methods =====================================================
+  
+  # Creates a new sorted array with an optional sort method supplied as
+  # a block. The sort method supplied should accept two arguments that are
+  # objects in the array to be compared and should return -1, 0, or 1 based
+  # on their sorting order. By default the comparison performed is <=>
+  def initialize(&sort_method)
+    sort_method ||= lambda { |a,b| a <=> b }
+    
+    @sort_method =
+      case (sort_method && sort_method.arity)
+      when 2
+        sort_method
+      when 1
+        lambda { |a,b| sort_method.call(a) <=> sort_method.call(b) }
+      else
+        raise SortArgumentRequired
+      end
+  end
+  
+  # Adds an object to the array by inserting it into the appropriate sorted
+  # location directly.
+  def <<(object)
+    low = 0
+    high = length
+    
+    while (low != high)
+      mid = (high + low) / 2
+      
+      comparison = @sort_method.call(object, at(mid))
+      
+      if (comparison < 0)
+        high = mid
+      elsif (comparison > 0)
+        low = mid + 1
+      else
+        break
+      end
+    end
+    
+    insert(low, object)
+  end
+  
+  # Combines another array with this one and returns the sorted result.
+  def +(array)
+    self.class[*super(array).sort(&@sort_method)]
+  end
+end

data/lib/pigeon/support.rb

@@ -1,3 +1,5 @@
+require 'digest/sha1'
+
 module Pigeon::Support
   # Uses the double-fork method to create a fully detached background
   # process. Returns the process ID of the created process. May throw an
@@ -30,6 +32,19 @@ module Pigeon::Support
     end
   end
     
+  # Returns a unique 160-bit identifier for this engine expressed as a 40
+  # character hexadecimal string. The first 32-bit sequence is a timestamp
+  # so these numbers increase over time and can be used to identify when
+  # a particular instance was launched.
+  def unique_id
+    '%8x%s' % [
+      Time.now.to_i,
+      Digest::SHA1.hexdigest(
+        '%.8f%8x' % [ Time.now.to_f, rand(1 << 32) ]
+      )[0, 32]
+    ]
+  end
+
   # Make all methods callable directly without having to include it
   extend self
 end

data/lib/pigeon/task.rb

@@ -0,0 +1,188 @@
+class Pigeon::Task
+  # == Constants ============================================================
+  
+  # == Properties ===========================================================
+  
+  attr_reader :state
+  attr_reader :engine
+  attr_reader :exception
+  attr_reader :created_at
+  attr_reader :started_at
+
+  # == Class Methods ========================================================
+
+  # Defines the initial state of this type of task. Default is :initialized
+  # but this can be customized in a subclass.
+  def self.initial_state
+    :initialized
+  end
+  
+  # Returns an array of the terminal states for this task. Default is
+  # :failed, :finished but this can be customized in a subclass.
+  def self.terminal_states
+    @terminal_states ||= [ :failed, :finished ].freeze
+  end
+
+  # == Instance Methods =====================================================
+  
+  def initialize(engine = nil)
+    @engine = engine || Pigeon::Engine.default_engine
+    @created_at = Time.now
+    
+    after_initialized
+  end
+  
+  # Kicks off the task. An optional callback is executed just before each
+  # state is excuted and is passed the state name as a symbol.
+  def run!(engine = nil, &callback)
+    @engine = engine if (engine)
+    @callback = callback if (block_given?)
+    
+    @state = self.class.initial_state
+    @started_at = Time.now
+
+    run_state!(@state)
+  end
+
+  # Returns true if the task is in the finished state, false otherwise.
+  def finished?
+    @state == :finished
+  end
+
+  # Returns true if the task is in the failed state, false otherwise.
+  def failed?
+    @state == :failed
+  end
+  
+  # Returns true if an exception was thrown, false otherwise.
+  def exception?
+    !!@exception
+  end
+  
+  # Returns true if the task is in any terminal state.
+  def terminal_state?
+    self.class.terminal_states.include?(@state)
+  end
+  
+  # Dispatches a block to be run as soon as possible.
+  def dispatch(&block)
+    @engine.dispatch(&block)
+  end
+  
+  # Returns a numerical priority order. If redefined in a subclass,
+  # should return a comparable value.
+  def priority
+    @created_at
+  end
+  
+  def inspect
+    "<#{self.class}\##{self.object_id}>"
+  end
+  
+  def <=>(task)
+    self.priority <=> task.priority
+  end
+  
+protected
+  def run_state!(state)
+    # Grab the current state and save it here, as it may switch at any time
+    @state = state
+    terminate = self.class.terminal_states.include?(state)
+
+    before_state(state)
+
+    send_callback(state) if (@callback)
+    
+    unless (terminate)
+      state_method = :"state_#{state}!"
+
+      # Only perform this state action if it is defined, otherwise ignore
+      # as some states may be deliberately NOOP in order to wait for some
+      # action to be completed asynchronously.
+      if (respond_to?(state_method))
+        send(state_method)
+      end
+    end
+    
+  rescue Object => e
+    @exception = e
+
+    handle_exception(e) rescue nil
+    
+    transition_to_state(:failed) unless (self.failed?)
+    
+    self.after_failed
+    self.after_terminated
+  ensure
+    after_state(state)
+
+    if (terminate)
+      self.after_finished
+      
+      # Send a final notification callback
+      if (@callback and @callback.arity == 0)
+        @callback.call
+      end
+      
+      self.after_terminated
+    end
+  end
+
+  # Schedules the next state to be executed. This method should only be
+  # called once per state or it may result in duplicated state actions.
+  def transition_to_state(state)
+    @engine.dispatch do
+      run_state!(state)
+    end
+    
+    state
+  end
+
+  def send_callback(state)
+    # State-notificaton callbacks are not made to blocks that do not take
+    # arguments, but instead a singe final callback is made.
+    case (@callback.arity)
+    when 2
+      @callback.call(self, state)
+    when 1
+      @callback.call(state)
+    end
+  end
+  
+  # Called just after the task is initialized.
+  def after_initialized
+  end
+  
+  # Called before a particular state is executed.
+  def before_state(state)
+  end
+
+  # Called after a particular state is executed.
+  def after_state(state)
+  end
+
+  # Called just after the task is finished.
+  def after_finished
+  end
+
+  # Called just after the task fails.
+  def after_failed
+  end
+  
+  # Called after the task finishes or terminates.
+  def after_terminated
+  end
+
+  # Called when an exception is thrown during processing with the exception
+  # passed as the first argument. Default behavior is to do nothing but
+  # this can be customized in a subclass. Any exceptions thrown by this
+  # method are ignored.
+  def handle_exception(exception)
+  end
+  
+  # This defines the behaivor of the intialized state. By default this
+  # simply transitions to the finished state.
+  def state_initialized!
+    transition_to_state(:finished)
+  end
+end