sweatshop 1.5.2 → 1.6.0

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
-:build: 
-:minor: 5
-:patch: 2
+:patch: 0
 :major: 1
+:build: 
+:minor: 6

data/lib/message_queue/base.rb

@@ -8,5 +8,6 @@ module MessageQueue
     def delete(queue);        end
     def client;               end
     def stop;                 end
+    def flush_all(queue);     end
   end
 end

data/lib/message_queue/rabbit.rb

@@ -7,44 +7,38 @@ module MessageQueue
     end
 
     def delete(queue)
-      send_command do
-        client.queue(queue).delete
-      end
+      cmd(queue, :delete)
     end
 
     def queue_size(queue)
-      send_command do
-        client.queue(queue).message_count
-      end
+      cmd(queue, :message_count)
     end
 
     def enqueue(queue, data)
-      send_command do
-        client.queue(queue, :durable => true).publish(Marshal.dump(data), :persistent => true)
-      end
+      cmd(queue, :publish, Marshal.dump(data), :persistent => true)
     end
 
     def dequeue(queue)
-      send_command do
-        task = client.queue(queue).pop(:ack => true)
-        return unless task
-        Marshal.load(task)
-      end
+      task = cmd(queue, :pop, :ack => true)
+      return unless task
+      Marshal.load(task)
     end
 
     def confirm(queue)
-      send_command do
-        client.queue(queue).ack
-      end
+      cmd(queue, :ack)
+    end
+
+    def flush_all(queue)
+      cmd(queue, :purge)
     end
 
-    def send_command(&block)
+    def cmd(queue, command, *args)
       retried = false
       begin
-        block.call
+        client.queue(queue, :durable => true).send(command, *args)
       rescue Carrot::AMQP::Server::ServerDown => e
         if not retried
-          puts "Error #{e.message}. Retrying..."
+          Sweatshop.log "Error #{e.message}. Retrying..."
           @client = nil
           retried = true
           retry

data/lib/sweatshop.rb

@@ -44,6 +44,7 @@ module Sweatshop
           worker.stop
         end
         queue.stop
+        @stop.call if @stop.kind_of?(Proc)
         exit 
       end
       sleep 1 if wait
@@ -87,8 +88,12 @@ module Sweatshop
     @daemon = true
   end
 
-  def stop
-    @stop = true
+  def stop(&block)
+    if block_given?
+      @stop = block
+    else
+      @stop = true
+    end
   end
 
   def stop?
@@ -125,6 +130,12 @@ module Sweatshop
     @queue_groups ||= (workers.collect{|w| w.queue_group.to_s} << 'default').uniq
   end
 
+  def flush_all_queues
+    workers.each do |worker|
+      worker.flush_queue
+    end
+  end
+
   def pp_sizes
     max_width = workers.collect{|w| w.to_s.size}.max
     puts '-' * (max_width + 10)
@@ -155,6 +166,10 @@ module Sweatshop
     nil
   end
 
+  def enabled?
+    !!config['enable']
+  end
+
   def log(msg)
     return if logger == :silent
     logger ? logger.debug(msg) : puts(msg)
@@ -174,5 +189,5 @@ module Sweatshop
 end
 
 if defined?(RAILS_ROOT)
-  Dir.glob(RAILS_ROOT + '/app/workers/*.rb').each{|worker| require worker }
+  Dir.glob(RAILS_ROOT + '/app/workers/*.rb').each{|worker| require_or_load worker }
 end

data/lib/sweatshop/daemoned.rb

@@ -49,10 +49,6 @@ module Daemoned
           options[:ontop] = true
         end
 
-        opt.on('--instances=NUM', 'Allow multiple instances to run simultaneously? 0 for infinite. default: 1') do |value|
-          self.instances = value.to_i
-        end
-
         opt.on('--log-file=LOGFILE', 'Logfile to log to') do |value|
           options[:log_file] = File.expand_path(value)
         end

data/lib/sweatshop/metaid.rb

@@ -3,3 +3,9 @@ def meta_eval(&blk); metaclass.instance_eval(&blk); end
 def meta_def(name, &blk)
   meta_eval { define_method name, &blk }
 end
+
+def check_arity!(method, args)
+  arity = method.arity
+  valid = arity < 0 ? args.size >= arity.abs - 1 : args.size == arity
+  raise ArgumentError.new("#{method.name} expects #{arity} arguments") unless valid
+end

data/lib/sweatshop/sweatd.rb

@@ -34,10 +34,8 @@ module Sweatshop
     end
 
     sig(:hup) do
-      begin
-        puts "Received HUP"
-        Sweatshop.stop
-      ensure
+      puts "Received HUP"
+      Sweatshop.stop do
         remove_pid!
         puts "Restarting sweatd with #{start_cmd}..."
         `#{start_cmd}`        

data/lib/sweatshop/worker.rb

@@ -8,12 +8,10 @@ module Sweatshop
 
     def self.method_missing(method, *args, &block)
       if method.to_s =~ /^async_(.*)/
-        method        = $1
-        expected_args = instance.method(method).arity
-        if expected_args != args.size
-          raise ArgumentError.new("#{method} expects #{expected_args} arguments")
-        end
-        return instance.send(method, *args) unless config['enable']
+        method = $1
+        check_arity!(instance.method(method), args)
+
+        return instance.send(method, *args) unless async?
 
         uid  = ::Digest::MD5.hexdigest("#{name}:#{method}:#{args}:#{Time.now.to_f}")
         task = {:args => args, :method => method, :uid => uid, :queued_at => Time.now.to_i}
@@ -29,6 +27,10 @@ module Sweatshop
       end
     end
 
+    def self.async?
+      Sweatshop.enabled?
+    end
+
     def self.instance
       @instance ||= new
     end
@@ -41,6 +43,10 @@ module Sweatshop
       @queue_name ||= self.to_s
     end
 
+    def self.flush_queue
+      queue.flush_all(queue_name)
+    end
+
     def self.delete_queue
       queue.delete(queue_name)
     end
@@ -82,6 +88,16 @@ module Sweatshop
       rescue Exception => e
         log("Caught Exception: #{e.message}, \n#{e.backtrace.join("\n")}")
         call_exception_handler(e)
+
+        # the only way to re-queue messages with rabbitmq is to close and reopen the connection
+        # putting a 'sleep 2' in here to give the administrator to fix peristent problems, otherwise
+        # we'll hit an infinite loop
+        #
+        # THIS CODE IS PROBLEMATIC --- we need to put these tasks into a 'failed' queue so we don't run into infinite loops
+        # will just 'confirm' for now
+        #queue.stop
+        #sleep 2
+        confirm
       end
     end
 

data/test/hello_worker.rb

@@ -1,4 +1,4 @@
-require File.dirname(__FILE__) + '/../lib/sweatshop'
+require File.expand_path(File.dirname(__FILE__) + '/../lib/sweatshop')
 class HelloWorker < Sweatshop::Worker
   TEST_FILE = File.dirname(__FILE__) + '/test.txt' unless defined?(TEST_FILE)
 

data/test/test_functional_worker.rb

@@ -1,6 +1,6 @@
-require File.dirname(__FILE__) + '/../lib/sweatshop'
-require File.dirname(__FILE__) + '/test_helper'
-require File.dirname(__FILE__) + '/hello_worker'
+require File.expand_path(File.dirname(__FILE__) + '/../lib/sweatshop')
+require File.expand_path(File.dirname(__FILE__) + '/test_helper'     )
+require File.expand_path(File.dirname(__FILE__) + '/hello_worker'    )
 
 class WorkerTest < Test::Unit::TestCase
 
@@ -15,10 +15,7 @@ class WorkerTest < Test::Unit::TestCase
   end
 
   should "daemonize" do
-    begin
-      Sweatshop.config['enable'] = true
-      Sweatshop.logger = :silent
-  
+    enable_server do
       HelloWorker.async_hello('Amos')
   
       worker = File.expand_path(File.dirname(__FILE__) + '/hello_worker')
@@ -29,26 +26,19 @@ class WorkerTest < Test::Unit::TestCase
   
       File.delete('sweatd.log') if File.exist?('sweatd.log')
       assert_equal 'Hi, Amos', File.read(HelloWorker::TEST_FILE)
-  
-    rescue Exception => e
-      puts e.message
-      puts e.backtrace.join("\n")
-      fail "\n\n*** Functional test failed, is the rabbit server running on localhost? ***\n"
     end
   end
 
   should "connect to fallback servers if the default one is down" do
-    begin
-      Sweatshop.logger = :silent
-      Sweatshop.config['enable'] = true
+    enable_server do
       Sweatshop.config['default']['cluster'] =
         [
          'localhost:5671', # invalid
          'localhost:5672'  # valid
         ]
       HelloWorker.async_hello('Amos')
-  
-      assert_equal 'Amos', HelloWorker.dequeue[:args].first
+      task = HelloWorker.dequeue
+      assert_equal 'Amos', task[:args].first
 
       HelloWorker.queue.client = nil
 
@@ -61,12 +51,29 @@ class WorkerTest < Test::Unit::TestCase
   
       HelloWorker.async_hello('Amos')
       assert_equal 'Amos', HelloWorker.dequeue[:args].first
-  
+    end
+  end
+
+  should "exception handler" do
+    exception = nil
+    HelloWorker.on_exception do |e|
+      exception = e
+    end
+
+    HelloWorker.do_task(nil)
+    assert_equal NoMethodError, exception.class
+  end
+
+
+  def enable_server
+    Sweatshop.config['enable'] = true
+    Sweatshop.logger = :silent
+    begin
+      yield
     rescue Exception => e
       puts e.message
       puts e.backtrace.join("\n")
       fail "\n\n*** Functional test failed, is the rabbit server running on localhost? ***\n"
     end
   end
-
 end

data/test/test_sweatshop.rb

@@ -1,5 +1,5 @@
-require File.dirname(__FILE__) + '/test_helper'
-require File.dirname(__FILE__) + '/../lib/sweatshop'
+require File.expand_path(File.dirname(__FILE__) + '/test_helper'     )
+require File.expand_path(File.dirname(__FILE__) + '/../lib/sweatshop')
 
 class SweatshopTest < Test::Unit::TestCase
   Sweatshop.workers = []
@@ -25,13 +25,6 @@ class SweatshopTest < Test::Unit::TestCase
     assert_equal "Hi, Amos", worker.hello('Amos')
   end
 
-  should "assign a uid" do
-    Sweatshop.logger = :silent
-    Sweatshop.config['enable'] = false
-    uid = HelloWorker.async_hello('Amos')
-    assert_not_nil uid
-  end
-
   should "have before task" do
     HelloWorker.before_task do
       "hello"
@@ -46,18 +39,6 @@ class SweatshopTest < Test::Unit::TestCase
     assert_equal "goodbye", HelloWorker.after_task.call
   end
 
-  should "exception handler" do
-    Sweatshop.logger = :silent
-
-    exception = nil
-    HelloWorker.on_exception do |e|
-      exception = e
-    end
-
-    HelloWorker.do_task(nil)
-    assert_equal NoMethodError, exception.class
-  end
-
   should "chain before tasks" do
     MESSAGES = []
     class BaseWorker < Sweatshop::Worker

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: sweatshop
 version: !ruby/object:Gem::Version 
-  version: 1.5.2
+  hash: 15
+  prerelease: false
+  segments: 
+  - 1
+  - 6
+  - 0
+  version: 1.6.0
 platform: ruby
 authors: 
 - Amos Elliston
@@ -9,19 +15,25 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-03 00:00:00 -08:00
+date: 2011-04-05 00:00:00 -07:00
 default_executable: sweatd
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: carrot
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        - 7
+        - 0
         version: 0.7.0
-    version: 
+  type: :runtime
+  version_requirements: *id001
 description: See summary
 email: amos@geni.com
 executables: 
@@ -56,31 +68,38 @@ files:
 - test/test_functional_worker.rb
 - test/test_helper.rb
 - test/test_sweatshop.rb
+- bin/sweatd
 has_rdoc: true
 homepage: http://github.com/famoseagle/sweatshop
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Sweatshop is a simple asynchronous worker queue build on top of rabbitmq/ampq