micro_q 0.9.5 → 0.9.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ce5b78c270ba5ed5145cc8ebdb0fc8155cafaf8
-  data.tar.gz: caafc65fac3531906ec9d37e5f475e8d5c1bb560
+  metadata.gz: 285d44726c946a46f1781e23c1b96de900a355ce
+  data.tar.gz: 7b57f333f09f40d8d7f2f257c58233269f64b8e4
 SHA512:
-  metadata.gz: f1c625a377f9c3abd1b242f9e270e1a0acbad20edcde60e03f4f9cccee1b86236278cff444ebd057629bb206ea728346b3ec79249090cb8543ff3474faa2e63c
-  data.tar.gz: 17e5790a270f4476dfee6a93476b094609b6b9dbc5fa90f7da6fb83219477c0a6faad6db112b1ec1ef90bdb60883dd4e09660ee2942a90581e38f07d704c9106
+  metadata.gz: 8cfa95f52fe7d18cce6cb9fe1790bb970f59c9cd6fc2011705d4b0471dd5d104b6a2631a0ebaf93921ea1d5a5f3d71abad283113542e24f538af797a1ac7f8c3
+  data.tar.gz: eea48e42cc2c63d1d051cb784ca044340e58ae458f50fda04026661bf75997f05ab5b9c6532823944f6022ebeb4243b4a5d480fdd78911a4db9048be8dd9e60f

data/README.md

@@ -39,7 +39,7 @@ end
 ###Simple
 
 ```ruby
-Called on the class invoked on an instance.
+# Called on the class invoked on an instance.
 
 MyWorker.perform_async
 MyWorker.update_async(:user_id => user.id)
@@ -102,9 +102,15 @@ MicroQ.configure do |config|
   config.queue = MicroQ::Queue::Sqs
   config.aws = { :key => 'YOUR KEY', :secret => 'YOUR SECRET' }
 end
+```
 
 **Note that when using the SQS Queue only the MicroQ's started via command-line will actually process messages**
 
+Start via the command line to process messages `microq`
+
+This queue also sets up thress named queues `:low`, `:default` and `:critical`
+
+```ruby
 # Then just use the queues in your workers
 class SomeWorker
   worker :queue => :critical

data/lib/micro_q.rb

@@ -33,6 +33,10 @@ module MicroQ
     manager.queue.push(*args)
   end
 
+  def self.queue_only?
+    @queue_only ||= MicroQ.config.sqs? && !MicroQ.config.worker_mode?
+  end
+
   private
 
   def self.manager
@@ -44,7 +48,7 @@ module MicroQ
   end
 
   def self.clear
-    @config = @manager = nil
+    @config = @manager = @queue_only = nil
   end
 end
 

data/lib/micro_q/cli.rb

@@ -14,41 +14,67 @@ module MicroQ
       opts = Slop.parse do
         banner 'Usage: microq [options]'
 
+        on 'h', 'This menu'
+        on 'help', 'This menu'
+        on 't=', 'The queue type to process [redis, sqs]'
+        on 'type=', 'The queue type to process [redis, sqs]'
         on 'r=', 'The path to the rails application'
         on 'require=', 'The path to the rails application'
         on 'w=', 'The number of worker threads'
         on 'workers=', 'The number of worker threads'
       end
 
+      (puts usage; exit) if opts[:help] || opts[:h]
+
       @workers = opts[:workers] || opts[:w]
-      @require = opts[:require] || opts[:r]
+      @require = opts[:require] || opts[:r] || 'config/environment.rb'
+      @mode = (opts[:type] || opts[:t] || 'sqs'.tap { puts 'Defaulting to sqs type' }).downcase
     end
 
     def verify!
-      raise "Need a valid path to a rails application, you gave us #{@require}\n" unless /environment\.rb/ === @require || File.exist?("#{@require}/config/application.rb")
+      unless File.exist?(@require) || File.exist?("#{@require}/config/application.rb")
+        puts 'Typically the -r option is simply the base path of the application'
+        puts 'This could be `pwd` or something special in production...'
+        puts "We tried both File.exist?(opts[:require]) and File.exist?(opts[:require] + 'config/application.rb')"
+        raise "MicroQ requires a valid rails application path via the -r option\n"
+      end
     end
 
     def setup
-      puts 'Requiring rails...'
+      puts 'Requiring rails application...'
+
       require 'rails'
 
-      puts 'Requiring rails application...'
       if File.directory?(@require)
         require File.expand_path("#{@require}/config/environment.rb")
       else
         require @require
       end
 
+      Rails.application.eager_load!
+
       aws_keys = MicroQ.config.aws.try(:keys) || []
-      raise 'SQS mode requires an aws :key and :secret see https://github.com/bnorton/micro_q/wiki/Named-Queues' unless aws_keys.include?(:key) && aws_keys.include?(:secret)
+
+      queue = if @mode == 'sqs'
+        raise 'SQS mode requires an aws :key and :secret see https://github.com/bnorton/micro_q/wiki/Named-Queues' unless aws_keys.include?(:key) && aws_keys.include?(:secret)
+
+        MicroQ::Queue::Sqs
+      elsif @mode == 'redis'
+       puts 'Using redis config' + MicroQ.config.redis.map {|(k,v)| "#{k}=#{v}" }.join(' ')
+
+        MicroQ::Queue::Redis
+      else
+        raise 'Only Redis and SQS mode are supported via the command line.'
+      end
 
       MicroQ.configure do |config|
-        config.queue = MicroQ::Queue::Sqs # set workers after since this must set workers=0 internally
+        config.queue = queue # set workers after assigning the queue since this sets workers=0 internally for the sqs queue
         config.workers = @workers.to_i if @workers
         config['worker_mode?'] = true
       end
 
-      puts "Running micro_q in SQS mode with #{MicroQ.config.workers} workers... Hit ctl+c to stop...\n"
+      puts "Running micro_q in #{@mode.upcase} mode with #{MicroQ.config.workers} workers..."
+      puts 'ctl+c to shutdown...'
       MicroQ.start
 
       sleep
@@ -56,5 +82,16 @@ module MicroQ
       puts 'Exiting via interrupt'
       exit(1)
     end
+
+    def usage
+      <<-USAGE
+  Run microq with messages enqueued to and dequeued from SQS or Redis.
+  Usage: microq [options]
+    -h, --help              This menu.
+    -t, --type [redis, sqs] The type of queue being utilized (either Redis or SQS)
+    -r, --require           The path the rails app's config/environment.rb file.
+    -w, --workers           The number of workers for this process.
+      USAGE
+    end
   end
 end

data/lib/micro_q/fetchers/sqs.rb

@@ -9,6 +9,10 @@ module MicroQ
         @manager = manager
       end
 
+      ##
+      # Long poll the SQS messages API and if there are messages
+      # then fetch more right away. Send messages to the manager
+      # when they return from the API
       def start
         defer do
           client.messages.tap do |messages|
@@ -16,22 +20,35 @@ module MicroQ
           end
         end
 
-        after(2) { start }
+        after(SHORT_DELAY) { start }
       end
 
+      ##
+      # Add the message to the sqs queue
+      # Respect the maximum amount of time that a message can
+      # be delayed (900 seconds).
+      #
       def add_message(message, time=nil)
-        message['run_at'] = time.to_f if time
+        message['run_at'] = [time.to_f, (Time.now + 900).to_i].min if time
 
         defer do
           client.messages_create(message)
         end
       end
 
+      def remove_message(message)
+        defer do
+          client.messages_delete(message)
+        end
+      end
+
       private
 
       def client
         @client ||= MicroQ::SqsClient.new(name)
       end
+
+      SHORT_DELAY = 2
     end
   end
 end

data/lib/micro_q/manager/default.rb

@@ -20,19 +20,16 @@ module MicroQ
       # Invoke this when the Queue or Worker pool dies
       exit_handler :reinitialize
 
-      attr_reader :queue, :workers
+      attr_reader :queue, :workers, :busy, :current
 
       def start
-        return if queue_only?
+        return if MicroQ.queue_only?
 
         count = workers.size
 
         if (messages = queue.dequeue(count)).any?
           messages.each do |message|
-            worker = workers.pop
-            @busy << worker
-
-            worker.perform!(message)
+            work_on(message)
           end
         end
 
@@ -40,22 +37,41 @@ module MicroQ
       end
 
       def work_done(worker)
-        @busy.delete(worker)
+        message = current.delete(worker)
+        queue.finished!(message) if queue.respond_to?(:finished)
+
+        busy.delete(worker)
         workers.push(worker)
       end
 
+      def work_on(message)
+        worker = workers.pop
+        busy << worker
+
+        current[worker] = message
+
+        worker.perform!(message)
+      end
+
       ##
       # Handle init/death of the Queue or the Worker pool
+      # When a worker dies the args are (#<Actor ...>, #<Exception>)
       #
-      def reinitialize(*)
+      def reinitialize(*args)
         kill_all and return if self.class.shutdown?
 
-        unless @queue && @queue.alive?
+        unless @queue && queue.alive?
           @queue = MicroQ.config.queue.new_link
         end
 
         @busy ||= []
         @workers ||= []
+        @current ||= {}
+
+        if args.any?
+          message = current.delete(args.first)
+          queue.finished!(message) if queue.respond_to?(:finished)
+        end
 
         build_missing_workers
       end
@@ -64,10 +80,10 @@ module MicroQ
 
       # Don't shrink the pool if the config changes
       def build_missing_workers
-        return if queue_only?
+        return if MicroQ.queue_only?
 
         workers.select!(&:alive?)
-        @busy.select!(&:alive?)
+        busy.select!(&:alive?)
 
         missing_worker_count.times do
           workers << MicroQ.config.worker.new_link(current_actor)
@@ -75,15 +91,11 @@ module MicroQ
       end
 
       def missing_worker_count
-        [MicroQ.config.workers - (workers.size + @busy.size), 0].max
+        [MicroQ.config.workers - (workers.size + busy.size), 0].max
       end
 
       def kill_all
-        (@workers + @busy).each {|w| w.terminate if w.alive? }
-      end
-
-      def queue_only?
-        @queue_only ||= MicroQ.config.sqs? && !MicroQ.config.worker_mode?
+        (workers + busy).each {|w| w.terminate if w.alive? }
       end
 
       def self.shutdown?

data/lib/micro_q/middleware/server/retry.rb

@@ -23,19 +23,39 @@ module MicroQ
           retried!(message)
           stats(message)
 
-          MicroQ.push(message, message['retried'])
+          unless message['retried']['count'] > retry_count(message)
+            MicroQ.push(message, message['retried'])
+          end
 
           raise e
         end
 
         private
 
+        def retry_count(msg)
+          [Fixnum, String].include?(msg['retry'].class) ? msg['retry'].to_i : 25
+        end
+
         def retried!(msg)
           msg['retried'] ||= { 'count' => 0 }
 
-          msg['retried']['count'] += 1
           msg['retried']['at']    = Time.now
-          msg['retried']['when']  = (Time.now + 15).to_f
+          msg['retried']['when']  = next_retry(msg['retried'])
+          msg['retried']['count'] += 1
+        end
+
+        ## On retry:
+        #    0-3   random interval up to 12 seconds
+        #    4-13  retry after an exponential amount of time
+        #          otherwise retry after 8 hours
+        def next_retry(retried)
+          count = retried['count']
+
+          (Time.now + case count
+            when 0..3 then rand(2..12)
+            when 4..13 then (count** 4)
+            else 8*60*60
+          end).to_f
         end
 
         def stats(msg)

data/lib/micro_q/queue/sqs.rb

@@ -11,14 +11,14 @@ module MicroQ
       def initialize
         @lock = Mutex.new
 
-        @messages, @entries, @later = [], [], []
+        @messages, @fetchers, @entries, @later = [], [], [], []
         @fetcher_map = {}
 
         build_missing_fetchers
       end
 
-      def push(item)
-        async.sync_push(item)
+      def push(*args)
+        async.sync_push(*args)
       end
 
       def sync_push(item, options={})
@@ -26,7 +26,7 @@ module MicroQ
         item['class'] = item['class'].to_s
 
         MicroQ.middleware.client.call(item, options) do
-          args, queue_name = [item], verify_queue(item['queue'].to_s)
+          args, queue_name = [item], verify_queue(item['queue'])
 
           if (time = options['when'])
             args << time.to_f
@@ -52,8 +52,13 @@ module MicroQ
         end
       end
 
+      def finished(item)
+        queue_name = verify_queue(item['queue'])
+        @fetcher_map[queue_name].remove_message(item)
+      end
+
       def verify_queue(name)
-        QUEUES_KEYS.include?(name) ? name : 'default'
+        QUEUES_KEYS.include?(name.to_s) ? name.to_s : 'default'
       end
 
       def self.shutdown!
@@ -73,7 +78,7 @@ module MicroQ
           ((existing = @fetcher_map[name]) && existing.alive? && existing) ||
             MicroQ::Fetcher::Sqs.new_link(name, current_actor).tap do |fetcher|
               @fetcher_map[name] = fetcher
-              fetcher.start!
+              fetcher.start! unless MicroQ.queue_only?
             end
         end
       end

data/lib/micro_q/sqs_client.rb

@@ -35,6 +35,13 @@ module MicroQ
       client.send_message(attrs)[:message_id]
     end
 
+    def messages_delete(message)
+      client.delete_message(
+        :queue_url => message['sqs_queue'],
+        :receipt_handle => message['sqs_handle']
+      )
+    end
+
     private
 
     def client

data/lib/micro_q/version.rb

@@ -1,7 +1,7 @@
 module MicroQ
   MAJOR = 0
   MINOR = 9
-  POINT = 5
+  POINT = 6
 
   VERSION = [MAJOR, MINOR, POINT].join('.')
 end

data/spec/lib/fetchers/sqs_spec.rb

@@ -76,6 +76,27 @@ describe MicroQ::Fetcher::Sqs do
 
         add_message
       end
+
+      describe 'when the time is too far in the future' do
+        let(:add_message) { subject.add_message(message, Time.now.to_i + 1000) }
+
+        it 'should truncate to 900 seconds from now' do
+          @client.should_receive(:messages_create).with(message.merge('run_at' => Time.now.to_i + 900))
+
+          add_message
+        end
+      end
+    end
+  end
+
+  describe '#remove_message' do
+    let(:message) { {'class' => 'FooBar'} }
+    let(:remove_message) { subject.remove_message(message) }
+
+    it 'should create the message' do
+      @client.should_receive(:messages_delete).with(message)
+
+      remove_message
     end
   end
 end

data/spec/lib/manager/default_spec.rb

@@ -83,6 +83,13 @@ describe MicroQ::Manager::Default do
         subject.start
       end
 
+      it 'should add the items to the currently working items' do
+        subject.start
+
+        subject.current[@worker1].should == @other_item
+        subject.current[@worker2].should == @item
+      end
+
       describe 'when the manager is in SQS mode' do
         before do
           MicroQ.config['sqs?'] = true
@@ -118,10 +125,12 @@ describe MicroQ::Manager::Default do
   end
 
   describe '#reinitialize' do
-    let(:death) { -> { subject.reinitialize } }
+    let(:current) { subject.current }
+
+    let(:death) { -> { subject.reinitialize(@worker2, Exception.new('worker2 crashed')) } }
 
     before do
-      @queue = mock(MicroQ::Queue::Default, :alive? => true, :dequeue => [])
+      @queue = mock(MicroQ::Queue::Default, :alive? => true, :dequeue => [], :finished! => nil, :respond_to? => true)
       MicroQ::Queue::Default.stub(:new_link).and_return(@queue)
 
       @worker1 = mock(MicroQ::Worker::Standard, :alive? => true, :perform! => nil)
@@ -178,7 +187,10 @@ describe MicroQ::Manager::Default do
       end
 
       describe 'when a busy worker has died' do
+        let(:message) { mock('message') }
+
         before do
+          current[@worker2] = message
           subject.wrapped_object.instance_variable_set(:@busy, [@worker2])
         end
 
@@ -199,6 +211,31 @@ describe MicroQ::Manager::Default do
 
           subject.workers.should == [@worker1, @new_worker2]
         end
+
+        it 'should remove the worker from current' do
+          current.should have_key(@worker2)
+          death.call
+
+          current.should_not have_key(@worker2)
+        end
+
+        it 'should finish the message with the queue' do
+          @queue.should_receive(:finished!).with(message)
+
+          death.call
+        end
+
+        describe 'when the queue does not respond to finished' do
+          before do
+            @queue.stub(:respond_to?).with(:finished).and_return(false)
+          end
+
+          it 'should not call it' do
+            @queue.should_not_receive(:finished!)
+
+            death.call
+          end
+        end
       end
 
       describe 'when in SQS mode' do
@@ -215,4 +252,57 @@ describe MicroQ::Manager::Default do
       end
     end
   end
+
+  describe '#work_done' do
+    let(:message) { mock('message') }
+    let(:current) { subject.current }
+
+    let(:work_done) { subject.work_done(@worker) }
+
+    before do
+      @worker = mock('worker')
+      @queue = mock(MicroQ::Queue::Default, :respond_to? => true, :finished! => nil)
+      MicroQ::Queue::Default.stub(:new_link).and_return(@queue)
+
+      subject.wrapped_object.instance_variable_set(:@busy, [@worker])
+      current[@worker] = message
+    end
+
+    it 'should remove the worker from busy' do
+      work_done
+
+      subject.busy.should == []
+    end
+
+    it 'should add the worker to workers' do
+      work_done
+
+      subject.workers.should include(@worker)
+    end
+
+    it 'should remove the worker from current' do
+      current.should have_key(@worker)
+      work_done
+
+      current.should_not have_key(@worker)
+    end
+
+    it 'should finish the message with the queue' do
+      @queue.should_receive(:finished!).with(message)
+
+      work_done
+    end
+
+    describe 'when the queue does not respond to finished' do
+      before do
+        @queue.stub(:respond_to?).with(:finished).and_return(false)
+      end
+
+      it 'should not call it' do
+        @queue.should_not_receive(:finished!)
+
+        work_done
+      end
+    end
+  end
 end

data/spec/lib/middleware/server/retry_spec.rb

@@ -34,6 +34,14 @@ describe MicroQ::Middleware::Server::Retry, :middleware => true do
             call
           }.to raise_error(Exception, 'an exception')
         end
+
+        it 'should not push the message' do
+          MicroQ.should_not_receive(:push)
+
+          expect {
+            call
+          }.to raise_error(Exception, 'an exception')
+        end
       end
 
       describe 'when retry is enabled' do
@@ -79,12 +87,15 @@ describe MicroQ::Middleware::Server::Retry, :middleware => true do
         end
 
         it 'should update the retry when time' do
-          Timecop.freeze(DateTime.now) do
-            safe(:call); @payload['retried']['when'].to_i.should == (Time.now + 15).to_i
+          @payload['retried'] = { 'count' => 1 }
 
-            Timecop.travel(100)
-            safe(:call); @payload['retried']['when'].to_i.should == (Time.now + 15).to_i
-          end
+          expect {
+            safe(:call)
+          }.to change { @payload['retried']['when'] }
+
+          expect {
+            safe(:call)
+          }.to change { @payload['retried']['when'] }
         end
 
         it 'should set the last retry time' do
@@ -105,10 +116,64 @@ describe MicroQ::Middleware::Server::Retry, :middleware => true do
 
         it 'should enqueue for the next retry time' do
           Timecop.freeze do
-            MicroQ.should_receive(:push).with(anything, hash_including('when' => (Time.now + 15).to_f))
+            MicroQ.should_receive(:push).with(anything, hash_including('when'))
+
+            safe(:call)
+          end
+        end
+
+        describe 'when the message has retried 25 times' do
+          before do
+            @payload['retried'] = { 'count' => 25 }
+          end
+
+          it 'should not push the message' do
+            MicroQ.should_not_receive(:push)
+
+            expect {
+              call
+            }.to raise_error(Exception, 'an exception')
+          end
+        end
+
+        describe 'when the message should retry 10 times' do
+          before do
+            @payload['retry'] = 10
+          end
+
+          it 'should retry the message' do
+            MicroQ.should_receive(:push)
 
             safe(:call)
           end
+
+          describe 'when the message has retried 10 times' do
+            before do
+              @payload['retried'] = { 'count' => 10 }
+            end
+
+            it 'should not push the message' do
+              MicroQ.should_not_receive(:push)
+
+              expect {
+                call
+              }.to raise_error(Exception, 'an exception')
+            end
+
+            describe 'when the retry count is a string' do
+              before do
+                @payload['retry'] = '10'
+              end
+
+              it 'should not push the message' do
+                MicroQ.should_not_receive(:push)
+
+                expect {
+                  call
+                }.to raise_error(Exception, 'an exception')
+              end
+            end
+          end
         end
       end
     end

data/spec/lib/queue/sqs_spec.rb

@@ -30,6 +30,18 @@ describe MicroQ::Queue::Sqs do
     it 'should have the fetchers' do
       subject.fetchers.uniq.should == [@fetcher]
     end
+
+    describe 'when we are in queue only mode' do
+      before do
+        MicroQ.stub(:queue_only? => true)
+      end
+
+      it 'should not start the fetcher' do
+        @fetcher.should_not_receive(:start!)
+
+        subject
+      end
+    end
   end
 
   describe '#receive_messages' do
@@ -203,4 +215,28 @@ describe MicroQ::Queue::Sqs do
       end
     end
   end
+
+  describe '#finished' do
+    before do
+      @fetchers = [:low, :default, :critical].collect do |name|
+        mock('MicroQ::Fetcher::Sqs : ' + name.to_s, :start! => nil).tap do |fetcher|
+          MicroQ::Fetcher::Sqs.stub(:new_link).with(name.to_s, anything).and_return(fetcher)
+        end
+      end
+    end
+
+    [:low, :default, :critical].each_with_index do |name, i|
+      describe "when the message has a queue named #{name}" do
+        before do
+          item['queue'] = name
+        end
+
+        it 'should create the message on the right queue' do
+          @fetchers[i].should_receive(:remove_message).with(item)
+
+          subject.finished(item)
+        end
+      end
+    end
+  end
 end

data/spec/lib/sqs_client_spec.rb

@@ -163,4 +163,27 @@ describe MicroQ::SqsClient, :aws => true do
       end
     end
   end
+
+  describe '#messages_delete' do
+    let(:message) { { 'sqs_handle' => 'handle-123', 'sqs_queue' => 'http://654.queue' } }
+    let(:messages_delete) { subject.messages_delete(message) }
+
+    it 'should delete the message' do
+      @client.should_receive(:delete_message)
+
+      messages_delete
+    end
+
+    it 'should have the queue url' do
+      @client.should_receive(:delete_message).with(hash_including(:queue_url => 'http://654.queue'))
+
+      messages_delete
+    end
+
+    it 'should have the message handle' do
+      @client.should_receive(:delete_message).with(hash_including(:receipt_handle => 'handle-123'))
+
+      messages_delete
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: micro_q
 version: !ruby/object:Gem::Version
-  version: 0.9.5
+  version: 0.9.6
 platform: ruby
 authors:
 - Brian Norton
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-04 00:00:00.000000000 Z
+date: 2013-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: celluloid