Bira-sweat_shop 1.0.0

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2009-01-13
+
+* 1 major enhancement
+
+  * Birthday!
+

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Amos Elliston
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,70 @@
+# SweatShop
+
+SweatShop provides an api to background resource intensive tasks. Much of the api design was copied from Workling, with a few tweaks.
+Currently, it runs rabbitmq and kestrel, but it can support any number of queues.
+
+## Installing 
+
+    gem install sweat_shop
+    freeze in your rails directory
+    cd vendor/gems/sweat_shop
+    rake setup
+
+## Writing workers
+
+Put `email_worker.rb` into app/workers and sublcass `SweatShop::Worker`:
+
+    class EmailWorker
+      def send_mail(to)
+        user = User.find_by_id(to)
+        Mailer.deliver_welcome(to)
+      end
+    end
+
+Then, anywhere in your app you can execute:
+
+    EmailWorker.async_send_mail(1)
+
+The `async` signifies that this task will be placed on a queue to be serviced by the EmailWorker possibly on another machine. You can also
+call:
+
+    EmailWorker.send_mail(1)
+
+That will do the work immediately, without placing the task on the queue. You can also define a `queue_group` at the top of the file
+which will allow you to split workers out into logical groups. This is important if you have various machines serving different
+queues. 
+
+## Running the queue
+
+SweatShop has been tested with Rabbit and Kestrel, but it will also work with Starling. Please use the following resources to install the server:
+
+Kestrel:
+http://github.com/robey/kestrel/tree/master
+
+Rabbit:
+http://github.com/ezmobius/nanite/tree/master
+
+config/sweatshop.yml specifies the machine address of the queue (default localhost:5672). You can also specify the queue type with the queue param.
+
+## Running the workers
+
+Assuming you ran `rake setup` in Rails, you can type:
+
+    script/sweatshop
+
+By default, the script will run all workers defined in the app/workers dir. Every task will be processed on each queue using a round-robin algorithm. You can also add the `-d` flag which will put the worker in daemon mode. The daemon also takes other params.  Add a `-h` for more details.
+
+    script/sweatshop -d
+    script/sweatshop -d stop
+
+If you would like to run SweatShop as a daemon on a linux machine, use the initd.sh script provided in the sweat_shop/script dir.
+
+# REQUIREMENTS
+
+    i_can_daemonize
+    memcache (for kestrel)
+    carrot (for rabbit)
+
+# LICENSE
+
+Copyright (c) 2009 Amos Elliston, Geni.com; Published under The MIT License, see License

data/Rakefile

@@ -0,0 +1,42 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rcov/rcovtask'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "sweat_shop"
+    s.summary = %Q{SweatShop is a simple asynchronous worker queue build on top of rabbitmq/ampq}
+    s.email = "amos@geni.com"
+    s.homepage = "http://github.com/famoseagle/sweat-shop"
+    s.description = "TODO"
+    s.authors = ["Amos Elliston"]
+    s.files =  FileList["[A-Z]*", "{lib,test,config}/**/*"]
+    s.add_dependency('famoseagle-carrot', '= 0.6.0')
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+Rake::TestTask.new
+
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'new_project'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+Rcov::RcovTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = true
+end
+
+task :default => :test
+
+task :setup do
+  require File.dirname(__FILE__) + '/install'
+end

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:major: 1
+:minor: 0
+:patch: 0

data/bin/sweatd

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/sweat_shop/sweatd'

data/config/defaults.yml

@@ -0,0 +1,9 @@
+# default options
+default: 
+  queue: rabbit
+  host: localhost
+  port: 5672
+  user: 'guest'
+  pass: 'guest'
+  vhost: '/'
+enable: true

data/config/sweatshop.yml

@@ -0,0 +1,18 @@
+development:
+  default:
+    queue: rabbit
+    host: localhost 
+    port: 5672
+  enable: true
+test:
+  default:
+    queue: rabbit
+    host: localhost 
+    port: 5672
+  enable: true
+production:
+  default:
+    queue: rabbit
+    host: localhost 
+    port: 5672
+  enable: true

data/lib/message_queue/base.rb

@@ -0,0 +1,17 @@
+module MessageQueue
+  class Base
+    attr_reader :opts
+    def queue_size(queue);    end
+    def enqueue(queue, data); end
+    def dequeue(queue);       end
+    def confirm(queue);       end
+    def subscribe(queue);     end
+    def delete(queue);        end
+    def client;               end
+    def stop;                 end
+
+    def subscribe?
+      false
+    end
+  end
+end

data/lib/message_queue/kestrel.rb

@@ -0,0 +1,32 @@
+module MessageQueue
+  class Kestrel < Base
+    def initialize(opts)
+      @servers = opts['servers']
+    end
+
+    def queue_size(queue)
+      size  = 0
+      stats = client.stats
+      servers.each do |server|
+        size += stats[server]["queue_#{queue}_items"].to_i
+      end
+      size
+    end
+
+    def enqueue(queue, data)
+      client.set(queue, data)
+    end
+
+    def dequeue(queue)
+      client.get("#{queue}/open")
+    end
+
+    def confirm(queue)
+      client.get("#{queue}/close")
+    end
+
+    def client
+      @client ||= MemCache.new(servers) 
+    end
+  end
+end

data/lib/message_queue/rabbit.rb

@@ -0,0 +1,71 @@
+require 'carrot'
+module MessageQueue
+  class Rabbit < Base
+
+    def initialize(opts={})
+      @opts = opts
+    end
+
+    def delete(queue)
+      send_command do
+        client.queue(queue).delete
+      end
+    end
+
+    def queue_size(queue)
+      send_command do
+        client.queue(queue).message_count
+      end
+    end
+
+    def enqueue(queue, data)
+      send_command do 
+        client.queue(queue, :durable => true).publish(Marshal.dump(data), :persistent => true)
+      end
+    end
+
+    def dequeue(queue)
+      send_command do
+        task = client.queue(queue).pop(:ack => true)
+        return unless task
+        Marshal.load(task)
+      end
+    end
+
+    def confirm(queue)
+      send_command do
+        client.queue(queue).ack
+      end
+    end
+
+    def send_command(&block)
+      retried = false
+      begin
+        block.call
+      rescue Carrot::AMQP::Server::ServerDown => e
+        if not retried
+          puts "Error #{e.message}. Retrying..."
+          @client = nil
+          retried = true
+          retry
+        else
+          raise e
+        end
+      end
+    end
+
+    def client
+      @client ||= Carrot.new(
+        :host  => @opts['host'], 
+        :port  => @opts['port'].to_i, 
+        :user  => @opts['user'], 
+        :pass  => @opts['pass'], 
+        :vhost => @opts['vhost']
+      ) 
+    end
+
+    def stop
+      client.stop
+    end
+  end
+end

data/lib/sweat_shop/daemoned.rb

@@ -0,0 +1,405 @@
+require 'optparse'
+require 'timeout'    
+
+module Daemoned
+  class DieTime      < StandardError; end
+  class TimeoutError < StandardError; end
+
+  def self.included(base)
+    base.extend ClassMethods           
+    base.initialize_options
+  end
+
+  class Config
+    METHODS = [:script_path]
+    CONFIG = {}    
+    def method_missing(name, *args)
+      name = name.to_s.upcase.to_sym
+      if name.to_s =~ /^(.*)=$/
+        name = $1.to_sym
+        CONFIG[name] = args.first
+      else
+        CONFIG[name]
+      end
+    end    
+  end
+
+  module ClassMethods  
+    
+    def initialize_options    
+      @@config             = Config.new
+      @@config.script_path = File.expand_path(File.dirname($0))
+      $0                   = script_name
+    end
+    
+    def parse_options
+      opts = OptionParser.new do |opt|
+        opt.banner = "Usage: #{script_name} [options] [start|stop]"
+
+        opt.on_tail('-h', '--help', 'Show this message') do
+          puts opt
+          exit(1)
+        end
+
+        opt.on('--loop-every=SECONDS', 'How long to sleep between each loop') do |value|
+          options[:loop_every] = value 
+        end
+
+        opt.on('-t', '--ontop', 'Stay on top (does not daemonize)') do
+          options[:ontop] = true
+        end
+
+        opt.on('--instances=NUM', 'Allow multiple instances to run simultaneously? 0 for infinite. default: 1') do |value|
+          self.instances = value.to_i
+        end
+
+        opt.on('--log-file=LOGFILE', 'Logfile to log to') do |value|
+          options[:log_file] = File.expand_path(value)
+        end
+
+        opt.on('--pid-file=PIDFILE', 'Location of pidfile') do |value|
+          options[:pid_file] = File.expand_path(value)
+        end
+
+        opt.on('--no-log-prefix', 'Do not prefix PID and date/time in log file.') do
+          options[:log_prefix] = false
+        end
+      end
+      
+      extra_args.each do |arg|
+        opts.on(*arg.first) do |value|
+          arg.last.call(value) if arg.last
+        end
+      end
+
+      opts.parse!
+      
+      if ARGV.include?('stop')                                                         
+        stop
+      elsif ARGV.include?('reload')
+        kill('HUP')
+        exit
+      elsif not ARGV.include?('start') and not ontop?
+        puts opts.help
+      end
+    end    
+
+    def arg(*args, &block)
+      self.extra_args << [args, block]
+    end
+
+    def extra_args
+      @extra_args ||= [] 
+    end
+
+    def callbacks
+      @callbacks ||= {}
+    end
+
+    def options
+      @options ||= {}
+    end
+
+    def options=(options)
+      @options = options
+    end
+
+    def config
+      yield @@config
+    end
+    
+    def before(&block)
+      callbacks[:before] = block
+    end
+
+    def after(&block)
+      callbacks[:after] = block
+    end
+
+    def sig(*signals, &block)
+      signals.each do |s|
+        callbacks["sig_#{s}".to_sym] = block
+      end
+    end
+
+    def die_if(method=nil,&block)
+      options[:die_if] = method || block
+    end
+
+    def exit_if(method=nil,&block)
+      options[:exit_if] = method || block
+    end
+
+    def callback!(callback)
+      callbacks[callback].call if callbacks[callback]
+    end
+
+    # options may include:
+    #
+    # <tt>:loop_every</tt> Fixnum (DEFAULT 0)
+    #  How many seconds to sleep between calls to your block
+    #
+    # <tt>:timeout</tt> Fixnum (DEFAULT 0)
+    #  Timeout in if block does not execute withing passed number of seconds
+    #
+    # <tt>:kill_timeout</tt> Fixnum (DEFAULT 120)
+    #  Wait number of seconds before using kill -9 on daemon
+    #
+    # <tt>:die_on_timeout</tt> BOOL (DEFAULT False)
+    #  Should the daemon continue running if a block times out, or just run the block again
+    #
+    # <tt>:ontop</tt> BOOL (DEFAULT False)
+    #  Do not daemonize.  Run in current process
+    #
+    # <tt>:before</tt> BLOCK
+    #  Run this block after daemonizing but before begining the daemonize loop.
+    #  You can also define the before block by putting a before do/end block in your class.
+    #
+    # <tt>:after</tt> BLOCK
+    #  Run this block before program exists.  
+    #  You can also define the after block by putting an after do/end block in your class.
+    #
+    # <tt>:die_if</tt> BLOCK
+    #  Run this check after each iteration of the loop.   If the block returns true, throw a DieTime exception and exit
+    #  You can also define the after block by putting an die_if do/end block in your class.
+    #      
+    # <tt>:exit_if</tt> BLOCK
+    #  Run this check after each iteration of the loop.   If the block returns true, exit gracefully
+    #  You can also define the after block by putting an exit_if do/end block in your class.
+    #
+    # <tt>:log_prefix</tt> BOOL (DEFAULT true)
+    #  Prefix log file entries with PID and timestamp
+    def daemonize(opts={}, &block)
+      self.options = opts
+      parse_options
+      return unless ok_to_start?
+
+      puts "Starting #{script_name}..."
+      puts "Logging to: #{log_file}" unless ontop?
+      
+      unless ontop?
+        safefork do
+          open(pid_file, 'w'){|f| f << Process.pid }
+          at_exit { remove_pid! }
+    
+          trap('TERM') { callback!(:sig_term)                            }
+          trap('INT')  { callback!(:sig_int)  ; Process.kill('TERM', $$) }
+          trap('HUP')  { callback!(:sig_hup)                             }
+
+          sess_id = Process.setsid
+          reopen_filehandes
+
+          begin
+            at_exit { callback!(:after) }
+            callback!(:before)
+            run_block(&block)
+          rescue SystemExit
+          rescue Exception => e
+            $stdout.puts "Something bad happened #{e.inspect} #{e.backtrace.join("\n")}"
+          end            
+        end
+      else
+        begin
+          callback!(:before)
+          run_block(&block)
+        rescue SystemExit, Interrupt
+          callback!(:after)
+        end
+      end
+    end
+
+  private
+    
+    def run_block(&block)
+      loop do
+        if options[:timeout]
+          begin
+            Timeout::timeout(options[:timeout].to_i) do
+              block.call if block              
+            end
+          rescue Timeout::Error => e
+            if options[:die_on_timeout]
+              raise TimeoutError.new("#{self} timed out after #{options[:timeout]} seconds while executing block in loop")
+            else
+              $stderr.puts "#{self} timed out after #{options[:timeout]} seconds while executing block in loop #{e.backtrace.join("\n")}"
+            end
+          end            
+        else
+          block.call if block
+        end
+
+        if options[:loop_every]
+          sleep options[:loop_every].to_i
+        elsif not block
+          sleep 0.1
+        end
+
+        break if should_exit?
+        raise DieTime.new('Die if conditions were met!') if should_die?
+      end                    
+      exit(0)
+    end
+
+    def should_die?
+      die_if = options[:die_if]
+      if die_if
+        if die_if.is_a?(Symbol) or die_if.is_a?(String)
+          self.send(die_if)
+        elsif die_if.is_a?(Proc)
+          die_if.call
+        end
+      else
+        false
+      end
+    end
+
+    def should_exit?
+      exit_if = options[:exit_if]
+      if exit_if
+        if exit_if.is_a?(Symbol) or exit_if.is_a?(String)
+          self.send(exit_if.to_sym)
+        elsif exit_if.is_a?(Proc)
+          exit_if.call
+        end
+      else
+        false
+      end
+    end
+
+    def ok_to_start?
+      return true if pid.nil?
+
+      if process_alive?
+        $stderr.puts "#{script_name} is already running"
+        return false          
+      else
+        $stderr.puts "Removing stale pid: #{pid}..."
+      end
+
+      true
+    end
+
+    def stop
+      puts "Stopping #{script_name}..."
+      kill
+      exit
+    end     
+
+    def kill(signal = 'TERM')
+      if pid.nil?
+        $stderr.puts "#{script_name} doesn't appear to be running"
+        exit(1)
+      end
+      $stdout.puts("Sending pid #{pid} signal #{signal}...")
+      begin
+        Process.kill(signal, pid)             
+        return if signal == 'HUP' 
+        if pid_running?(options[:kill_timeout] || 120)
+          $stdout.puts("Using kill -9 #{pid}")
+          Process.kill(9, pid)
+        else
+          $stdout.puts("Process #{pid} stopped")
+        end
+      rescue Errno::ESRCH
+       $stdout.puts("Couldn't #{signal} #{pid} as it wasn't running")
+      end
+    end               
+    
+    def pid_running?(time_to_wait = 0)
+      times_to_check = 1
+      if time_to_wait > 0.5
+        times_to_check = (time_to_wait / 0.5).to_i
+      end
+      times_to_check.times do
+        return false unless process_alive?
+        sleep 0.5
+      end
+      true
+    end
+          
+    def safefork(&block)
+      fork_tries ||= 0
+      fork(&block)
+    rescue Errno::EWOULDBLOCK
+      raise if fork_tries >= 20
+      fork_tries += 1
+      sleep 5
+      retry
+    end
+
+    def process_alive?
+      Process.kill(0, pid)
+      true
+    rescue Errno::ESRCH => e
+      false
+    end  
+
+    LOG_FORMAT  = '%-6d %-19s %s'
+    TIME_FORMAT = '%Y/%m/%d %H:%M:%S'
+    def reopen_filehandes
+      STDIN.reopen('/dev/null')
+      STDOUT.reopen(log_file, 'a')
+      STDOUT.sync = true          
+      STDERR.reopen(STDOUT)
+      if log_prefix?
+        def STDOUT.write(string)
+          if @no_prefix
+            @no_prefix = false if string[-1, 1] == "\n"
+          else
+            string = LOG_FORMAT % [$$, Time.now.strftime(TIME_FORMAT), string]
+            @no_prefix = true              
+          end
+          super(string)
+        end
+      end
+    end
+
+    def remove_pid!
+      if File.file?(pid_file) and File.read(pid_file).to_i == $$
+        File.unlink(pid_file)
+      end
+    end
+
+    def ontop?
+      options[:ontop]
+    end
+
+    def log_prefix?
+      options[:log_prefix] || true
+    end                    
+    
+    LOG_PATHS = ['log/', 'logs/', '../log/', '../logs/', '../../log', '../../logs', '.']
+    LOG_PATHS.unshift("#{RAILS_ROOT}/log") if defined?(RAILS_ROOT)
+    def log_dir
+      options[:log_dir] ||= begin
+        LOG_PATHS.detect do |path|
+          File.exists?(File.expand_path(path))        
+        end
+      end               
+    end
+    
+    def log_file
+      options[:log_file] ||= File.expand_path("#{log_dir}/#{script_name}.log")
+    end
+
+    def pid_dir
+      options[:pid_dir] ||= log_dir
+    end
+
+    def pid_file
+      options[:pid_file] ||= File.expand_path("#{pid_dir}/#{script_name}.pid")
+    end
+    
+    def pid
+      @pid ||= File.file?(pid_file) ? File.read(pid_file).to_i : nil
+    end
+
+    def script_name
+      @script_name ||= File.basename($0).gsub('.rb', '')
+    end
+
+    def script_name=(script_name)
+      @script_name = script_name
+    end
+  end
+end

data/lib/sweat_shop/metaid.rb

@@ -0,0 +1,5 @@
+def metaclass; class << self; self; end; end
+def meta_eval(&blk); metaclass.instance_eval(&blk); end
+def meta_def(name, &blk)
+  meta_eval { define_method name, &blk }
+end

data/lib/sweat_shop/sweatd.rb

@@ -0,0 +1,76 @@
+require File.dirname(__FILE__) + '/daemoned'
+
+module SweatShop
+  class Sweatd
+    include Daemoned
+    queues     = []
+    groups     = []
+    rails_root = nil
+    start_cmd  = "ruby #{__FILE__} start #{ARGV.reject{|a| a == 'start'}.join(' ')}"
+
+    arg '--workers=Worker,Worker', 'Workers to service (Default is all)' do |value|
+      queues = value.split(',')
+    end
+
+    arg '--groups=GROUP,GROUP', 'Groups of queues to service' do |value|
+      groups = value.split(',').collect{|g| g.to_sym}
+    end
+
+    arg '--worker-file=WORKERFILE', 'Worker file to load'  do |value|
+      require value
+    end
+
+    arg '--worker-dir=WORKERDIR', 'Directory containing workers'  do |value|
+      Dir.glob(value + '*.rb').each{|worker| require worker}
+    end
+
+    arg '--rails=DIR', 'Pass in RAILS_ROOT to run this daemon in a rails environment' do |value|
+      rails_root = value
+    end
+
+    sig(:term, :int) do
+      puts "Shutting down sweatd..."
+      SweatShop.stop
+    end
+
+    sig(:hup) do
+      puts "Received HUP"
+      SweatShop.stop
+      remove_pid!
+      puts "Restarting sweatd with #{start_cmd}..."
+      `#{start_cmd}`        
+    end
+    
+    before do
+      if rails_root
+        puts "Loading Rails..."
+        require rails_root + '/config/environment' 
+      end
+      require File.dirname(__FILE__) + '/../sweat_shop'
+    end
+
+    daemonize(:kill_timeout => 20) do
+      workers = []
+
+      if groups.any?
+        workers += SweatShop.workers_in_group(groups)
+      end
+
+      if queues.any?
+        workers += queues.collect{|q| Object.module_eval(q)}
+      end
+
+      if workers.any?
+        worker_str = workers.join(',')
+        puts "Starting #{worker_str}..." 
+        $0 = "Sweatd: #{worker_str}"
+        SweatShop.do_tasks(workers)
+      else
+        puts "Starting all workers..." 
+        $0 = 'Sweatd: all'
+        SweatShop.do_all_tasks
+      end
+    end
+
+  end
+end

data/lib/sweat_shop/worker.rb

@@ -0,0 +1,142 @@
+require File.dirname(__FILE__) + '/metaid'
+
+module SweatShop
+  class Worker
+    def self.inherited(subclass)
+      self.workers << subclass
+    end
+
+    def self.method_missing(method, *args, &block)
+      if method.to_s =~ /^async_(.*)/
+        method        = $1
+        expected_args = instance.method(method).arity
+        if expected_args != args.size 
+          raise ArgumentError.new("#{method} expects #{expected_args} arguments")
+        end
+        return instance.send(method, *args) unless config['enable']
+
+        uid  = ::Digest::MD5.hexdigest("#{name}:#{method}:#{args}:#{Time.now.to_f}")
+        task = {:args => args, :method => method, :uid => uid, :queued_at => Time.now.to_i}
+
+        log("Putting #{uid} on #{queue_name}")
+        enqueue(task)
+
+        uid
+      elsif instance.respond_to?(method)
+        instance.send(method, *args)
+      else
+        super
+      end
+    end
+
+    def self.instance
+      @instance ||= new
+    end
+
+    def self.config
+      SweatShop.config
+    end
+
+    def self.queue_name
+      @queue_name ||= self.to_s
+    end
+
+    def self.delete_queue
+      queue.delete(queue_name)
+    end
+
+    def self.queue_size
+      queue.queue_size(queue_name)
+    end
+
+    def self.enqueue(task)
+      queue.enqueue(queue_name, task)
+    end
+
+    def self.dequeue
+      queue.dequeue(queue_name)
+    end
+
+    def self.confirm
+      queue.confirm(queue_name)
+    end
+
+    def self.subscribe
+      queue.subscribe(queue_name) do |task|
+        do_task(task)
+      end
+    end
+    
+    def self.do_tasks
+      while task = dequeue
+        do_task(task)
+      end
+    end
+
+    def self.do_task(task)
+      call_before_task(task)
+
+      queued_at = task[:queued_at] ? "(queued #{Time.at(task[:queued_at]).strftime('%Y/%m/%d %H:%M:%S')})" : ''
+      log("Dequeuing #{queue_name}::#{task[:method]} #{queued_at}")
+      task[:result] = instance.send(task[:method], *task[:args])
+
+      call_after_task(task)
+      confirm
+    end
+
+    def self.call_before_task(task)
+      superclass.call_before_task(task) if superclass.respond_to?(:call_before_task)
+      before_task.call(task) if before_task
+    end
+
+    def self.call_after_task(task)
+      superclass.call_after_task(task) if superclass.respond_to?(:call_after_task)
+      after_task.call(task) if after_task
+    end
+
+    def self.queue
+      SweatShop.queue(queue_group.to_s)
+    end
+
+    def self.workers
+      SweatShop.workers
+    end
+
+    def self.config
+      SweatShop.config
+    end
+
+    def self.log(msg)
+      SweatShop.log(msg)
+    end
+
+    def self.before_task(&block)
+      if block
+        @before_task = block
+      else
+        @before_task
+      end
+    end
+
+    def self.after_task(&block)
+      if block
+        @after_task = block
+      else
+        @after_task
+      end
+    end
+
+    def self.stop
+      instance.stop
+    end
+
+    # called before we exit -- subclass can implement this method 
+    def stop; end;
+
+
+    def self.queue_group(group=nil)
+      group ? meta_def(:_queue_group){ group } : _queue_group
+    end
+    queue_group :default
+  end
+end

data/lib/sweat_shop.rb

@@ -0,0 +1,147 @@
+require 'rubygems'
+require 'digest'
+require 'yaml'
+
+$:.unshift(File.dirname(__FILE__))
+require 'message_queue/base'
+require 'message_queue/rabbit'
+require 'message_queue/kestrel'
+require 'sweat_shop/worker'
+
+module SweatShop
+  extend self
+
+  def workers
+    @workers ||= []
+  end
+
+  def workers=(workers)
+    @workers = workers 
+  end
+
+  def workers_in_group(groups)
+    groups = [groups] unless groups.is_a?(Array)
+    if groups.include?(:all)
+      workers
+    else
+      workers.select do |worker|
+        groups.include?(worker.queue_group)
+      end
+    end
+  end
+
+  def do_tasks(workers)
+    if queue.subscribe?
+      EM.run do
+        workers.each do |worker|
+          worker.subscribe
+        end
+      end
+    else
+      loop do
+        wait = true
+        workers.each do |worker|
+          if task = worker.dequeue
+            worker.do_task(task)
+            wait = false
+          end
+        end
+        if stop?
+          workers.each do |worker|
+            worker.stop
+          end
+          queue.stop
+          exit 
+        end
+        sleep 1 if wait
+      end
+    end
+  end
+
+  def do_all_tasks
+    do_tasks(
+      workers_in_group(:all)
+    )
+  end
+
+  def do_default_tasks
+    do_tasks(
+      workers_in_group(:default)
+    )
+  end
+
+  def config
+    @config ||= begin
+      defaults = YAML.load_file(File.dirname(__FILE__) + '/../config/defaults.yml')
+      if defined?(RAILS_ROOT)
+        file = RAILS_ROOT + '/config/sweatshop.yml'
+        if File.exist?(file)
+          YAML.load_file(file)[RAILS_ENV || 'development']
+        else
+          defaults['enable'] = false
+          defaults
+        end
+      else
+        defaults
+      end
+    end
+  end
+
+  def stop
+    @stop = true
+    queue.stop if queue.subscribe?
+  end
+
+  def stop?
+    @stop
+  end
+
+  def queue_sizes
+    workers.inject([]) do |all, worker|
+      all << [worker, worker.queue_size]
+      all
+    end
+  end
+
+  def pp_sizes
+    max_width = workers.collect{|w| w.to_s.size}.max
+    puts '-' * (max_width + 10)
+    puts queue_sizes.collect{|p| sprintf("%-#{max_width}s %2s", p.first, p.last)}.join("\n")
+    puts '-' * (max_width + 10)
+  end
+
+  def queue(type = 'default')
+    @queues ||= {}
+    @queues[type] ||= begin 
+      qconfig = config[type] || config['default']
+      qtype   = qconfig['queue'] || 'rabbit'
+      queue   = constantize("MessageQueue::#{qtype.capitalize}")
+      queue.new(qconfig)
+    end
+  end
+
+  def queue=(queue, type = 'default')
+    @queues[type] = queue
+  end
+
+  def log(msg)
+    return if logger == :silent
+    logger ? logger.debug(msg) : puts(msg)
+  end
+
+  def logger
+    @logger
+  end
+
+  def logger=(logger)
+    @logger = logger
+  end
+
+  def constantize(str)
+    Object.module_eval("#{str}", __FILE__, __LINE__)
+  end
+end
+
+if defined?(RAILS_ROOT)
+  Dir.glob(RAILS_ROOT + '/app/workers/*.rb').each{|worker| require worker }
+end

data/test/hello_worker.rb

@@ -0,0 +1,13 @@
+require File.dirname(__FILE__) + '/../lib/sweat_shop'
+class HelloWorker < SweatShop::Worker
+  TEST_FILE = File.dirname(__FILE__) + '/test.txt' unless defined?(TEST_FILE)
+
+  def hello(name)
+    puts name
+    "Hi, #{name}"
+  end
+
+  after_task do |task|
+    File.open(TEST_FILE, 'w'){|f| f << task[:result]}
+  end
+end

data/test/test_functional_worker.rb

@@ -0,0 +1,36 @@
+require File.dirname(__FILE__) + '/../lib/sweat_shop'
+require File.dirname(__FILE__) + '/test_helper'
+require File.dirname(__FILE__) + '/hello_worker'
+
+class WorkerTest < Test::Unit::TestCase
+
+  def setup
+    File.delete(HelloWorker::TEST_FILE) if File.exist?(HelloWorker::TEST_FILE)
+  end
+
+  def teardown
+    File.delete(HelloWorker::TEST_FILE) if File.exist?(HelloWorker::TEST_FILE)
+  end
+
+  test "daemon" do
+    begin
+      SweatShop.queue = nil
+      SweatShop.logger = :silent
+
+      worker = File.expand_path(File.dirname(__FILE__) + '/hello_worker')
+      sweatd = "#{File.dirname(__FILE__)}/../lib/sweat_shop/sweatd.rb" 
+      uid = HelloWorker.async_hello('Amos')
+
+      `ruby #{sweatd} --worker-file #{worker} start`
+      `ruby #{sweatd} stop`
+
+      File.delete('sweatd.log') if File.exist?('sweatd.log')
+      assert_equal 'Hi, Amos', File.read(HelloWorker::TEST_FILE)
+    rescue Exception => e
+      puts e.message
+      puts e.backtrace.join("\n")
+      fail "\n\n*** Functional test failed, is the rabbit server running on localhost? ***\n"
+    end
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,14 @@
+require 'test/unit'
+
+class << Test::Unit::TestCase
+  def test(name, &block)
+    test_name = "test_#{name.gsub(/[\s\W]/,'_')}"
+    raise ArgumentError, "#{test_name} is already defined" if self.instance_methods.include? test_name
+    define_method test_name, &block
+  end
+
+  def xtest(name, &block)
+    # no-op, an empty test method is defined to prevent "no tests in testcase" errors when all tests are disabled
+    define_method(:test_disabled) { assert true }
+  end
+end

data/test/test_sweatshop.rb

@@ -0,0 +1,65 @@
+require File.dirname(__FILE__) + '/test_helper'
+require File.dirname(__FILE__) + '/../lib/sweat_shop'
+
+class SweatShopTest < Test::Unit::TestCase
+  SweatShop.workers = []
+
+  class HelloWorker < SweatShop::Worker
+    def hello(name)
+      "Hi, #{name}"
+    end
+  end
+
+  class GroupedWorker < SweatShop::Worker
+    queue_group :foo
+  end
+
+  test "group workers" do
+    assert_equal [HelloWorker, GroupedWorker], SweatShop.workers_in_group(:all)
+    assert_equal [HelloWorker],   SweatShop.workers_in_group(:default)
+    assert_equal [GroupedWorker], SweatShop.workers_in_group(:foo)
+  end
+
+  test "synch call" do
+    worker = HelloWorker.new
+    assert_equal "Hi, Amos", worker.hello('Amos')
+  end
+
+  test "uid" do
+    SweatShop.logger = :silent
+    uid = HelloWorker.async_hello('Amos')
+    assert_not_nil uid
+  end
+
+  test "before task" do
+    HelloWorker.before_task do
+      "hello"
+    end
+    assert_equal "hello", HelloWorker.before_task.call
+  end
+
+  test "after task" do
+    HelloWorker.after_task do
+      "goodbye"
+    end
+    assert_equal "goodbye", HelloWorker.after_task.call
+  end
+
+  test "chainable before tasks" do
+    MESSAGES = []
+    class BaseWorker < SweatShop::Worker
+      before_task do |task|
+        MESSAGES << 'base'
+      end
+    end
+    class SubWorker < BaseWorker
+      before_task do |task|
+        MESSAGES << 'sub'
+      end
+    end
+    SubWorker.call_before_task('foo')
+    assert_equal ['base', 'sub'], MESSAGES
+    SweatShop.workers.delete(BaseWorker)
+    SweatShop.workers.delete(SubWorker)
+  end
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification 
+name: Bira-sweat_shop
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+platform: ruby
+authors: 
+- Amos Elliston
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-07-17 00:00:00 -07:00
+default_executable: sweatd
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: famoseagle-carrot
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 0.6.0
+    version: 
+description: TODO
+email: amos@geni.com
+executables: 
+- sweatd
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.markdown
+files: 
+- History.txt
+- LICENSE
+- README.markdown
+- Rakefile
+- VERSION.yml
+- config/defaults.yml
+- config/sweatshop.yml
+- lib/message_queue/base.rb
+- lib/message_queue/kestrel.rb
+- lib/message_queue/rabbit.rb
+- lib/sweat_shop.rb
+- lib/sweat_shop/daemoned.rb
+- lib/sweat_shop/metaid.rb
+- lib/sweat_shop/sweatd.rb
+- lib/sweat_shop/worker.rb
+- test/hello_worker.rb
+- test/test_functional_worker.rb
+- test/test_helper.rb
+- test/test_sweatshop.rb
+has_rdoc: true
+homepage: http://github.com/famoseagle/sweat-shop
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 2
+summary: SweatShop is a simple asynchronous worker queue build on top of rabbitmq/ampq
+test_files: 
+- test/hello_worker.rb
+- test/test_functional_worker.rb
+- test/test_helper.rb
+- test/test_sweatshop.rb