task_tempest 0.1.0 → 0.2.0

data/.gitignore

@@ -19,3 +19,4 @@ rdoc
 pkg
 
 ## PROJECT::SPECIFIC
+example/log

data/CHANGELOG

@@ -1,2 +1,4 @@
+0.2.0
+ - Refactored
 0.1.0
  - Initial version

data/Rakefile

@@ -51,3 +51,22 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
+
+namespace :example do
+  
+  desc "Run the example."
+  task :run do
+    `ruby example/my_tempest.rb run`
+  end
+  
+  desc "Fill the example queue."
+  task :fill do
+    require "example/my_tempest"
+    while true
+      r = rand
+      sleep(r)
+      MyTempest.submit([nil, "Evaler", %{sleep(#{r})}])
+    end
+  end
+  
+end

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/{examples → example}/my_tempest.rb

@@ -1,3 +1,4 @@
+Dir.chdir(File.dirname(__FILE__))
 $LOAD_PATH << "../lib"
 
 require "rubygems"
@@ -9,25 +10,44 @@ require "tasks/evaler"
 require "tasks/greeter"
 
 class MemcachedQueue
+  attr_reader :logger
   
-  def initialize(name)
+  def initialize(name, logger = nil)
     @name = name
+    @logger = logger || Logger.new(File.open("/dev/null", "w"))
     @cache = MemCache.new "localhost:11211"
   end
   
   def push(item)
+    logger.debug "MemcachedQueue#push #{item.inspect}"
     queue = @cache.fetch(@name){ [] }
     queue.push(item)
     @cache.set(@name, queue)
   end
   
+  alias_method :enqueue, :push
+  
   def pop
+    # @count ||= 0
+    # @count += 1
+    # if @count % 10 == 0
+    #   @count = 0
+    #   raise "pop failed"
+    # end
+    
     queue = @cache.fetch(@name){ [] }
     item = queue.pop
+    logger.debug "MemcachedQueue#pop #{item.inspect}" if item
     @cache.set(@name, queue)
     item
   end
   
+  def size
+    @cache.fetch(@name){ [] }.size
+  end
+  
+  alias_method :dequeue, :pop
+  
 end
 
 # To run this example, open two shells and navagate to the examples directory (i.e. the
@@ -41,20 +61,20 @@ end
 # Note this example requires the SystemTimer, daemons and memcache-client gems.
 class MyTempest < TaskTempest::Engine
   
-  # This dictates what the logs will be named.
-  process_name "my_tempest"
-  
   # How many threads.
-  threads 5
+  threads 3
   
   # Where to write the log files.
   log_dir "log"
   
+  # What to name the log files.
+  log_name "my_tempest"
+  
   # Where to look for task classes.  Will require each .rb file in this directory.
   task_dir "tasks"
   
   # Time in seconds between each bookkeeping event.
-  bookkeeping_interval 15
+  bookkeeping_interval 10
   
   # Don't display log messages below this level.
   log_level Logger::INFO
@@ -70,25 +90,6 @@ class MyTempest < TaskTempest::Engine
     MemcachedQueue.new("my_tempest_queue")
   end
   
-  # Define how to enqueue messages.  This is used by MyTempest.submit.
-  # message is a tuple [task_id, task_class_name, *task_arguments].
-  # *args are passed through from MyTempest.submit.
-  enqueue do |queue, message, logger, *args|
-    logger.debug "enqueue #{message.inspect}"
-    queue.push(message)
-  end
-  
-  # Define how to dequeue messages.  It must return either
-  # nil or a tuple: [task_id, task_class_name, *task_arguments]
-  dequeue do |queue, logger|
-    if (message = queue.pop)
-      logger.debug "dequeue #{message.inspect}"
-      message
-    else
-      nil
-    end
-  end
-  
   # Callback that happens after #init_logging, but before #bootstrap.
   before_initialize do |logger|
   end
@@ -104,7 +105,7 @@ class MyTempest < TaskTempest::Engine
   
   # Callback that happens when an exception occurs in a task.
   on_task_exception do |task, e, logger|
-    puts "(T:#{task_id}) #{e.class}: #{e.message}"
+    puts "(T:#{task.id}) #{e.class}: #{e.message}"
   end
   
   # Callback that happens when a task exceeds the task_timeout setting.
@@ -128,7 +129,7 @@ end
 
 if $0 == __FILE__
   require "daemons"
-  Daemons.run_proc(MyTempest.settings.process_name, :log_output => true) do
+  Daemons.run_proc(MyTempest.name, :log_output => true) do
     MyTempest.new.run
   end
 end

data/lib/task_tempest/active_support.rb

@@ -10,6 +10,30 @@ class Array
     end
     [passed, failed]
   end unless method_defined?(:separate)
+  
+  def avg(&block)
+    sum(&block).to_f / length
+  end
+end
+
+module Enumerable
+  def sum(identity = 0, &block)
+    return identity unless size > 0
+
+    if block_given?
+      map(&block).sum
+    else
+      inject { |sum, element| sum + element }
+    end
+  end unless method_defined?(:sum)
+end
+
+class Float
+  def round_with_task_tempest(precision = nil)
+    precision.nil? ? round_without_task_tempest : (self * (10 ** precision)).round / (10 ** precision).to_f
+  end
+  alias_method :round_without_task_tempest, :round
+  alias_method :round, :round_with_task_tempest
 end
 
 class Object

data/lib/task_tempest/bookkeeper.rb

@@ -1,63 +1,103 @@
 module TaskTempest
   class Bookkeeper
-    attr_reader :storm
+    attr_reader :logger
     
-    def initialize(storm)
-      @storm = storm
+    def initialize(options)
+      options.each{ |k, v| instance_variable_set("@#{k}", v) }
     end
     
-    def book
-      return @book if @book
+    def report(executions)
+      @timer      ||= Time.now
+      @executions ||= []
+      @executions += executions
       
-      book = {}
+      if Time.now - @timer > @interval
+        logger.info "[STATS] " + make_book.inspect
+        @executions.clear
+        @timer = Time.now
+      end
+    end
+    
+    def make_book
       
-      executions = storm.clear_executions(:finished?)
+      # Do some setup.
       ObjectSpace.garbage_collect
+      queue = @queue_factory.call
+      book = {}
+      
+      # Reset memoized objects.
+      @memory = nil
+      @files = nil
       
       # Task success/error counts.
       book[:tasks] = {}
-      book[:tasks][:total_count] = executions.length
-      book[:tasks][:error_count] = executions.inject(0){ |memo, e| memo += 1 if e.exception; memo }
-      book[:tasks][:error_percentage] = begin
-        if book[:tasks][:total_count] > 0
-          book[:tasks][:error_count].to_f / book[:tasks][:total_count] * 100.0
-        else
-          0.0
-        end
-      end
-      book[:tasks][:per_thread] = tasks_per_thread(storm.threads, executions).values
-      book[:tasks][:avg_duration] = executions.inject(0){ |memo, e| memo += e.duration; memo }.to_f / executions.length
+      book[:tasks][:counts] = task_counts
+      book[:tasks][:per_thread] = tasks_per_thread
+      book[:tasks][:durations] = task_durations
+      book[:tasks][:throughput] = task_throughput
       
       # Thread (worker) info.
       book[:threads] = {}
-      book[:threads][:busy] = storm.busy_workers.length
-      book[:threads][:idle] = storm.size - book[:threads][:busy]
-      book[:threads][:saturation] = book[:threads][:busy] / storm.size.to_f * 100 
+      book[:threads][:busy] = @storm.busy_workers.length
+      book[:threads][:idle] = @storm.size - book[:threads][:busy]
+      book[:threads][:saturation] = (book[:threads][:busy] / @storm.size.to_f * 100).round(2)
       
       # Memory, Object, GC info.
       book[:memory] = {}
       book[:memory][:live_objects] = ObjectSpace.live_objects rescue nil
-      book[:memory][:resident] = get_memory(:resident)
-      book[:memory][:virtual] = get_memory(:virtual)
+      book[:memory][:resident] = format_memory(get_memory(:resident))
+      book[:memory][:virtual] = format_memory(get_memory(:virtual))
       
       # Open file counts.
       book[:files] = {}
       book[:files][:total_count] = get_files(:total)
       book[:files][:tcp_count] = get_files(:tcp)
       
-      @book = book
+      # Queue info.
+      book[:queue] = {}
+      book[:queue][:size] = queue.size if queue.respond_to?(:size)
+      book[:queue][:backlog] = @storm.executions.inject(0){ |memo, e| memo += 1 unless e.started?; memo }
+      
+      book
+    end
+    
+    def task_counts
+      tot = @executions.length
+      err = @executions.sum{ |e| e.exception ? 1 : 0 }
+      pct = begin
+        if tot > 0
+          (err.to_f / tot)
+        else
+          0.0
+        end
+      end
+      { :tot => tot, :err => err, :pct => pct.round(3) }
+    end
+    
+    def task_throughput
+      duration = Time.now - @timer
+      per_sec = @executions.length.to_f / duration
+      per_min = (per_sec * 60).round(2)
+      "#{per_min}/m"
     end
     
-    # executions passed in are *finished*.
-    def tasks_per_thread(threads, executions)
-      counts_by_thread = threads.inject({}) do |memo, thread|
+    def tasks_per_thread
+      counts_by_thread = @storm.threads.inject({}) do |memo, thread|
         memo[thread] = 0
         memo
       end
-      executions.each do |e|
-        counts_by_thread[e.thread] += 1
+      @executions.each{ |e| counts_by_thread[e.thread] += 1 }
+      counts = counts_by_thread.values
+      { :min => counts.min, :max => counts.max, :avg => counts.avg.round(2) }
+    end
+    
+    def task_durations
+      durations = @executions.collect{ |execution| execution.duration }
+      if durations.length > 0
+        { :min => durations.min.round(3), :max => durations.max.round(3), :avg => durations.avg.round(3) }
+      else
+        "n/a"
       end
-      counts_by_thread
     end
     
     def get_memory(which)
@@ -83,5 +123,15 @@ module TaskTempest
       end
     end
     
+    KB = 1024
+    MB = KB**2
+    def format_memory(memory)
+      if memory > MB
+        (memory / MB).to_s + "M"
+      else
+        (memory / KB).to_s + "K"
+      end
+    end
+    
   end
 end

data/lib/task_tempest/bootstrap.rb

@@ -1,76 +1,152 @@
+require "thread_storm"
+require "task_tempest/bookkeeper"
+require "task_tempest/dispatcher"
+
 module TaskTempest
   module Bootstrap
     
-    def self.included(mod)
-      mod.send(:extend, ClassMethods)
-      mod.send(:include, InstanceMethods)
+    def logger
+      @logger ||= begin
+        log_name = settings.log_name || self.class.name
+        path = "#{settings.log_dir}/#{log_name}.log"
+        Logger.new(path).tap do |logger|
+          logger.formatter = LogFormatter
+          logger.level = settings.log_level
+        end
+      end
     end
     
-    module ClassMethods
+    def task_logger
+      @task_logger ||= begin
+        log_name = settings.log_name || self.class.name
+        path = "#{settings.log_dir}/#{log_name}.task.log"
+        Logger.new(path).tap do |logger|
+          logger.formatter = LogFormatter
+          logger.level = settings.log_level
+        end
+      end
     end
     
-    module InstanceMethods
-      
-    private
-      
-      def bootstrap
-        init_logging
-        with_error_handling(:halt_on_error) do
-          init_thread_pool
-          before_initialize
-          init_tasks
-          init_queue
-          after_initialize
-          init_require
+    def queue
+      @queue ||= begin
+        case settings.queue
+        when Proc
+          settings.queue.call(logger)
+        else
+          settings.queue
         end
       end
-      
-      def init_logging
-        @logger = Logger.new("#{settings.log_dir}/#{settings.process_name}.log")
-        @logger.formatter = LogFormatter
-        @logger.level = settings.log_level
-        logger.info "starting up"
-
-        @task_logger = Logger.new("#{settings.log_dir}/#{settings.process_name}.task.log")
-        @task_logger.formatter = LogFormatter
-        @task_logger.level = settings.log_level
+    end
+    
+    def storm
+      @storm ||= begin
+        ThreadStorm.new :size => settings.threads,
+                        :reraise => false,
+                        :execute_blocks => true,
+                        :timeout_method => settings.timeout_method,
+                        :timeout => settings.task_timeout
       end
-      
-      def init_tasks
+    end
+    
+    def dispatcher
+      @dispatcher ||= begin
+        Dispatcher.new :logger => logger,
+                       :task_logger => task_logger,
+                       :queue_factory => Proc.new{ settings.queue.call(logger) },
+                       :storm => storm,
+                       :no_message_sleep => settings.no_message_sleep
+      end
+    end
+    
+    def bookkeeper
+      @bookkeeper ||= begin
+        Bookkeeper.new :storm => storm,
+                       :queue_factory => Proc.new{ settings.queue.call(logger) },
+                       :interval => settings.bookkeeping_interval,
+                       :logger => logger
+      end
+    end
+    
+    def bootstrap(error_action)
+      init_logging
+      with_error_handling(error_action) do
+        before_initialize
+        init_require
+        init_tasks
+        init_thread_pool
+        init_queue
+        init_bookkeeper
+        init_task_logging
+        init_dispatcher
+        after_initialize
+      end
+    end
+    
+    def init_logging
+      @logger and return
+      logger
+      logger.info "logger initialized"
+    end
+    
+    def init_task_logging
+      @task_logger and return
+      task_logger
+      logger.info "task logger initialized"
+    end
+    
+    def init_thread_pool
+      @storm and return
+      logger.info "initializing thread pool"
+      storm
+    end
+    
+    def init_tasks
+      @init_tasks ||= begin
         logger.info "initializing tasks"
         Dir.glob("#{settings.task_dir}/*.rb").each do |file_path|
-          logger.info file_path
+          logger.debug file_path
           require file_path
         end
+        true
       end
-
-      def init_queue
-        logger.info "initializing queue"
-        @queue = settings.queue.call(logger)
-      end
-
-      def init_thread_pool
-        logger.info "initializing thread pool"
-        @storm = ThreadStorm.new :size => settings.threads,
-                                 :reraise => false,
-                                 :timeout_method => settings.timeout_method,
-                                 :timeout => settings.task_timeout
-      end
-      
-      def before_initialize
-        logger.info "calling before_initialize"
+    end
+    
+    def init_queue
+      @queue and return
+      logger.info "initializing queue"
+      queue
+    end
+    
+    def init_require
+      require "task_tempest/require"
+    end
+    
+    def init_bookkeeper
+      @bookkeeper and return
+      logger.info "initializing bookkeeper"
+      bookkeeper
+    end
+    
+    def init_dispatcher
+      @dispatcher and return
+      logger.info "initializing dispatcher"
+      dispatcher
+    end
+    
+    def before_initialize
+      @before_initialize ||= begin
+        logger.info "before_initialize called"
         settings.before_initialize.call(logger)
+        true
       end
-      
-      def after_initialize
-        logger.info "calling after_initialize"
+    end
+    
+    def after_initialize
+      @after_initialize ||= begin
         settings.after_initialize.call(logger)
+        logger.info "after_initialize called"
+        true
       end
-      
-      def init_require
-        require "task_tempest/require"
-      end
-      
     end
       
   end

data/lib/task_tempest/dispatcher.rb

@@ -0,0 +1,64 @@
+module TaskTempest
+  class Dispatcher
+    attr_reader :logger
+    
+    def initialize(options)
+      options.each{ |k, v| instance_variable_set("@#{k}", v) }
+      start
+    end
+    
+    def start
+      if dead?
+        @queue = @queue_factory.call
+        @thread = Thread.new{ run }
+      end
+    end
+    
+    alias_method :restart, :start
+    
+    def alive?
+      @thread and @thread.alive?
+    end
+    
+    def dead?
+      not alive?
+    end
+    
+    def exception
+      dead? and @thread.value
+    end
+    
+    def run
+      run_loop while true
+    end
+    
+    def run_loop
+      consume and dispatch
+    end
+    
+    def consume
+      @message = @queue.dequeue
+      if @message
+        true
+      else
+        logger.debug "queue empty, sleeping for #{@no_message_sleep}"
+        sleep(@no_message_sleep)
+        false
+      end
+    end
+    
+    def dispatch
+      task_id, task_class_name, *task_args = @message
+      task_class = TaskTempest::Task.const_get(task_class_name)
+      task = task_class.new(*task_args).init(:id => task_id, :logger => @task_logger)
+      task.execution = @storm.execute(task){ task.run }
+      logger.info task.format_log("started")
+    end
+    
+    def shutdown
+      @thread and @thread.kill
+      @thread.join
+    end
+    
+  end
+end

data/lib/task_tempest/engine.rb

@@ -1,6 +1,6 @@
+require "benchmark"
 require "thread_storm"
 
-require "task_tempest/bookkeeper"
 require "task_tempest/bootstrap"
 require "task_tempest/callbacks"
 require "task_tempest/error_handling"
@@ -8,7 +8,6 @@ require "task_tempest/settings"
 
 module TaskTempest
   class Engine
-    attr_reader :logger, :task_logger, :queue, :storm, :message, :tasks
     
     include Bootstrap
     include Callbacks
@@ -20,9 +19,7 @@ module TaskTempest
     end
     
     def self.submit_message(message, *args)
-      logger = Logger.new(STDOUT)
-      queue  = settings.queue.call(logger)
-      settings.enqueue.call(queue, message, logger, *args)
+      new.queue.enqueue(message, *args)
     end
     
     def self.submit_task(task, *args)
@@ -37,115 +34,61 @@ module TaskTempest
       end
     end
     
-    def initialize
-      @tasks = []
-      @bookkeeping_timer = Time.now
+    def self.run
+      new.run
     end
     
     def run
-      bootstrap
-      logger.info "starting run loop"
+      bootstrap(:halt)
       with_shutdown_handling{ heartbeat while true }
     end
     
   private
     
     def heartbeat
-      with_error_handling{ receive_message }
-      with_error_handling{ dispatch_message }
-      with_error_handling{ finish_tasks }
-      with_error_handling{ bookkeeping }
-    end
-    
-    def receive_message
-      logger.debug "receiving message"
-      
-      if message
-        logger.debug "already have message"
-        return
-      end
-      
-      # Why do we do it this way?  Because of badly behaved dequeue
-      # definitions.  For example, right_aws rescues any exception
-      # when making a request to Amazon.  Thus if we try to shutdown
-      # our tempest, right_aws could potentially swallow that exception.
-      
-      @receive_storm ||= ThreadStorm.new :size => 1,
-                                         :timeout_method => settings.timeout_method,
-                                         :timeout => settings.dequeue_timeout
-      
-      execution = @receive_storm.execute{ settings.dequeue.call(queue, logger) }
-      with_error_handling do
-        @message = execution.value
-        logger.warn "dequeue timed out" if execution.timed_out?
-      end
-      @receive_storm.clear_executions # Prevent memory leak.
-        
-      if message.nil?
-        logger.debug "no available messages, sleeping for #{settings.no_message_sleep}"
-        sleep(settings.no_message_sleep)
-      end
-    end
-    
-    def dispatch_message
-      if storm.busy_workers.length == storm.size
-        logger.debug "no available threads, sleeping for #{settings.no_thread_sleep}"
-        sleep(settings.no_thread_sleep)
-      elsif message
-        dispatch_task
+      time = Benchmark.realtime do
+        with_error_handling{ finish_tasks }
+        with_error_handling{ health_check }
+        with_error_handling{ bookkeeping }
       end
-    end
-    
-    def dispatch_task
-      id, name, *args = message
-      task = TaskTempest::Task.const_get(name).new(*args)
-      task.override :id => id, :logger => task_logger
-      task.spawn(storm)
-      tasks << task
-      logger.info task.format_log("started", true)
-      task.logger.info "arguments #{args.inspect}"
-    rescue Exception => e
-      raise
-    ensure
-      @message = nil # Ensure we pop a new message off the queue on next loop iteration.
+      logger.debug "heartbeat complete in #{time} seconds"
+      sleep(settings.pulse_delay)
     end
     
     def finish_tasks
-      finished, @tasks = tasks.separate{ |task| task.execution.finished? }
-      finished.each{ |task| handle_finished_task(task) }
+      @executions = storm.clear_executions(:finished?).each do |execution|
+        task = execution.args.first
+        if (e = execution.exception)
+          logger.info task.format_log("failed", true)
+          task.logger.fatal format_exception(e)
+          on_task_exception(task, e)
+        elsif execution.timed_out?
+          logger.info task.format_log("timed out", true)
+          on_task_timeout(task)
+        else
+          logger.info task.format_log("finished", true)
+          on_require(task, execution.value)
+        end
+      end
     end
     
-    def handle_finished_task(task)
-      if (e = task.execution.exception)
-        logger.info task.format_log("failed", true)
-        task.logger.fatal format_exception(e)
-        on_task_exception(task, e)
-      elsif task.execution.timed_out?
-        logger.info task.format_log("timed out", true)
-        on_task_timeout(task)
-      else
-        logger.info task.format_log("finished", true)
-        on_require(task, task.execution.value)
+    def health_check
+      if dispatcher.dead?
+        with_error_handling{ dispatcher.exception }
+        logger.error "dispatcher thread died, restarting"
+        dispatcher.restart
       end
     end
     
     def bookkeeping
-      # Return unless it's time to do bookkeeping.
-      if Time.now - @bookkeeping_timer > settings.bookkeeping_interval
-        @bookkeeping_timer = Time.now # Reset the timer.
-      else
-        return
-      end
-      
-      keeper = Bookkeeper.new(storm)
-      logger.info "[BOOKKEEPING] " + keeper.book.inspect
-      on_bookkeeping(keeper.book)
+      bookkeeper.report(@executions)
     end
     
     def clean_shutdown
-      logger.info "shutting down"
+      logger.info "shutting down..."
       begin
         timeout(settings.shutdown_timeout) do
+          dispatcher.shutdown
           storm.join
           storm.shutdown
         end
@@ -153,6 +96,7 @@ module TaskTempest
         logger.warn "shutdown timeout exceeded"
       end
       finish_tasks
+      logger.info "shutdown"
       exit(0)
     end
     

data/lib/task_tempest/error_handling.rb

@@ -7,17 +7,23 @@ module TaskTempest
       SignalException
     ]
     
-    def with_error_handling(halt_on_error = false)
+    def with_error_handling(error_action = :continue)
       yield
     rescue *SHUTDOWN_EXCEPTIONS => e
       raise
     rescue Exception => e
       on_internal_exception(e)
-      if halt_on_error
+      case error_action
+      when :halt
         logger.fatal format_exception(e)
         exit(-1)
-      else
+      when :reraise
+        logger.fatal format_exception(e)
+        raise
+      when :continue
         logger.error format_exception(e)
+      else
+        raise "Wtf man, typo."
       end
     end
     
@@ -45,7 +51,7 @@ module TaskTempest
     end
     
     def format_exception(e)
-      "#{e.class} #{e.message}\n" + e.backtrace.join("\n")
+      "#{e.class}: #{e.message}\n" + e.backtrace.join("\n")
     end
     
   end

data/lib/task_tempest/producer.rb

@@ -0,0 +1,58 @@
+require "thread"
+
+require "task_tempest/error_handling"
+
+module TaskTempest
+  class Producer
+    attr_reader :logger
+    
+    DEFAULTS = {
+      :delay => 1,
+      :buffer_size => 1
+    }
+    
+    def initialize(queue, logger, options = {})
+      options = DEFAULTS.merge(options)
+      @queue = queue
+      @logger = logger
+      @delay = options[:delay]
+      @buffer = SizedQueue.new(options[:buffer_size])
+      Thread.new{ run }
+    end
+    
+    def consume
+      if @buffer.size > 0
+        @buffer.pop
+      else
+        nil
+      end
+    end
+  
+  private
+  
+    def run
+      run_loop while true
+    end
+    
+    def run_loop
+      (dequeue and buffer) or delay
+    end
+    
+    def dequeue
+      @message = @queue.dequeue
+    rescue Exception => e
+      @message = e
+      nil
+    end
+    
+    def buffer
+      @buffer.push(@message)
+    end
+    
+    def delay
+      logger.debug "producer sleeping (queue empty) for #{@delay.inspect}"
+      sleep(@delay)
+    end
+    
+  end
+end

data/lib/task_tempest/settings.rb

@@ -5,22 +5,28 @@ module TaskTempest
   module Settings
     
     DEFAULTS = {
-      :process_name => "task_tempest",
-      :log_level => Logger::DEBUG,
+      # Basic settings
       :threads => 10,
-      :no_thread_sleep => 1,
+      :log_level => Logger::INFO,
+      :queue => nil,
+      :bookkeeping_interval => 10*60, # 10 minutes
+      :log_name => nil,
+      
+      # Delay settings
       :no_message_sleep => 1,
+      :pulse_delay => 0.25,
+      
+      # Timeout settings
+      :timeout_method => Timeout.method(:timeout),
       :task_timeout => nil,
       :shutdown_timeout => 5, # 5 seconds
-      :dequeue_timeout => 2, # 2 seconds
-      :timeout_method => Timeout.method(:timeout),
+      
+      # Directory settings
       :root_dir => File.expand_path(Dir.pwd),
       :log_dir => File.expand_path(Dir.pwd),
       :task_dir => File.expand_path(Dir.pwd),
-      :queue => nil,
-      :enqueue => Proc.new{ |queue, message| raise "not implemented" },
-      :dequeue => Proc.new{ |queue, logger| logger.error("dequeue not defined"); sleep(1); nil },
-      :bookkeeping_interval => 10*60, # 10 minutes
+      
+      # Callback settings
       :before_initialize => Proc.new{ |logger| },
       :after_initialize => Proc.new{ |logger| },
       :on_internal_exception => Proc.new{ |e, logger| },
@@ -47,10 +53,6 @@ module TaskTempest
     
     module ClassMethods
       
-      def process_name(value)
-        settings.process_name = value
-      end
-      
       def log_level(value)
         settings.log_level = value
       end
@@ -59,12 +61,16 @@ module TaskTempest
         settings.threads = value
       end
       
+      def log_name(value)
+        settings.log_name = value
+      end
+      
       def no_message_sleep(value)
         settings.no_message_sleep = value
       end
       
-      def no_thread_sleep(value)
-        settings.no_thread_sleep = value
+      def pulse_delay(value)
+        settings.pulse_delay = value
       end
       
       def root_dir(path)
@@ -95,16 +101,12 @@ module TaskTempest
         settings.shutdown_timeout = value.to_f
       end
       
-      def queue(&block)
-        settings.queue = block
-      end
-      
-      def enqueue(&block)
-        settings.enqueue = block
-      end
-      
-      def dequeue(&block)
-        settings.dequeue = block
+      def queue(queue = nil, &block)
+        if block_given?
+          settings.queue = block
+        else
+          settings.queue = queue
+        end
       end
       
       def bookkeeping_interval(value)

data/lib/task_tempest/task.rb

@@ -6,20 +6,18 @@ require "task_tempest/require"
 
 module TaskTempest
   class Task
-    attr_reader :id, :args, :execution
+    attr_reader :id, :args
+    attr_accessor :execution
     
     def initialize(*args)
       @id = generate_id
       @args = args
     end
     
-    def override(options = {})
+    def init(options = {})
       @id = options[:id] if options[:id]
       @logger = TaskLogger.new(options[:logger], self) if options[:logger]
-    end
-    
-    def spawn(storm)
-      @execution = storm.execute{ run }
+      self
     end
     
     def run
@@ -38,9 +36,9 @@ module TaskTempest
       [id, self.class.name, *args]
     end
     
-    def format_log(message, duration = false)
+    def format_log(message, show_duration = false)
       s = "{#{id}} <#{self.class}> #{message}"
-      s += " #{execution.duration}" if duration and execution.finished?
+      s += " #{execution.duration.round(3)}" if show_duration and execution.finished?
       s
     end
     

data/task_tempest.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{task_tempest}
-  s.version = "0.1.0"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Christopher J. Bottaro"]
-  s.date = %q{2010-06-24}
+  s.date = %q{2010-07-07}
   s.description = %q{Framework for creating queue based, threaded asychronous job processors.}
   s.email = %q{cjbottaro@alumni.cs.utexas.edu}
   s.extra_rdoc_files = [
@@ -24,16 +24,18 @@ Gem::Specification.new do |s|
      "README.rdoc",
      "Rakefile",
      "VERSION",
-     "examples/my_tempest.rb",
-     "examples/tasks/evaler.rb",
-     "examples/tasks/greeter.rb",
+     "example/my_tempest.rb",
+     "example/tasks/evaler.rb",
+     "example/tasks/greeter.rb",
      "lib/task_tempest.rb",
      "lib/task_tempest/active_support.rb",
      "lib/task_tempest/bookkeeper.rb",
      "lib/task_tempest/bootstrap.rb",
      "lib/task_tempest/callbacks.rb",
+     "lib/task_tempest/dispatcher.rb",
      "lib/task_tempest/engine.rb",
      "lib/task_tempest/error_handling.rb",
+     "lib/task_tempest/producer.rb",
      "lib/task_tempest/require.rb",
      "lib/task_tempest/settings.rb",
      "lib/task_tempest/task.rb",
@@ -45,21 +47,18 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/cjbottaro/task_tempest}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.3.6}
   s.summary = %q{Framework for creating asychronous job processors.}
   s.test_files = [
     "test/helper.rb",
-     "test/test_task_tempest.rb",
-     "examples/my_tempest.rb",
-     "examples/tasks/evaler.rb",
-     "examples/tasks/greeter.rb"
+     "test/test_task_tempest.rb"
   ]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
     else
     end
   else

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: task_tempest
 version: !ruby/object:Gem::Version 
-  hash: 27
   prerelease: false
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Christopher J. Bottaro
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-06-24 00:00:00 -05:00
+date: 2010-07-07 00:00:00 -05:00
 default_executable: 
 dependencies: []
 
@@ -36,16 +35,18 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
-- examples/my_tempest.rb
-- examples/tasks/evaler.rb
-- examples/tasks/greeter.rb
+- example/my_tempest.rb
+- example/tasks/evaler.rb
+- example/tasks/greeter.rb
 - lib/task_tempest.rb
 - lib/task_tempest/active_support.rb
 - lib/task_tempest/bookkeeper.rb
 - lib/task_tempest/bootstrap.rb
 - lib/task_tempest/callbacks.rb
+- lib/task_tempest/dispatcher.rb
 - lib/task_tempest/engine.rb
 - lib/task_tempest/error_handling.rb
+- lib/task_tempest/producer.rb
 - lib/task_tempest/require.rb
 - lib/task_tempest/settings.rb
 - lib/task_tempest/task.rb
@@ -63,33 +64,26 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: Framework for creating asychronous job processors.
 test_files: 
 - test/helper.rb
 - test/test_task_tempest.rb
-- examples/my_tempest.rb
-- examples/tasks/evaler.rb
-- examples/tasks/greeter.rb