sidekiq-sqs 0.0.11 → 0.0.17

data/README.md

@@ -30,7 +30,7 @@ That'll turn it all on. Nothing else _should_ have to change.
 
 ## Caveats
 
-* Scheduling is completely broken
+* Scheduling uses SQS message timers.  The amount of time you can schedule a worker must be 0 to 900 seconds (15 mins).
 * ?
 
 ## Contributing

data/lib/sidekiq-sqs/client.rb

@@ -35,15 +35,35 @@ module Sidekiq
       end
 
       module ClassMethods
+        def environmental_queue_name(queue_name)
+          return queue_name unless defined?(::Rails)
+
+          if queue_name =~ /^#{::Rails.env}_/
+            queue_name
+          else
+            %(#{::Rails.env}_#{queue_name})
+          end
+        end
+
+        def clear_queue(queue_name)
+          queue = queue_or_create(environmental_queue_name queue_name)
+
+          while message = queue.receive_message
+            message.delete
+          end
+
+          queue
+        end
+
         def push(item)
           normed = normalize_item(item)
           normed, payload = process_single(item['class'], normed)
 
           pushed = false
           if normed['at']
-            # FIXME - SQS only supports a delay of 15.minutes
-            # probably a schedule queue? or keep this in redis??
-            pushed = conn.zadd('schedule', normed['at'].to_s, payload)
+            delay_seconds = (normed['at'] - Time.now.to_f).round
+            raise "The number of seconds to delay should be from 0 to 900 (15 mins)." if delay_seconds > 900
+            pushed = queue_or_create(normed['queue']).send_message(payload, { delay_seconds: delay_seconds })
           else
             pushed = queue_or_create(normed['queue']).send_message(payload)
           end if normed
@@ -102,7 +122,7 @@ module Sidekiq
 
         def send_batch_to_sqs(queue, formatted_items)
           failures, retryables = [], []
-          
+
           begin
             queue.batch_send(formatted_items)
           rescue AWS::SQS::Errors::BatchSendError => error
@@ -113,11 +133,12 @@ module Sidekiq
             failures.concat failed
             retryables.concat retryable
           end
-          
+
           [failures, retryables]
         end
 
         def queue_or_create(queue)
+          queue = environmental_queue_name(queue)
           begin
             Sidekiq.sqs.queues.named(queue.to_s)
           rescue AWS::SQS::Errors::NonExistentQueue

data/lib/sidekiq-sqs/fetcher.rb

@@ -1,6 +1,8 @@
 module Sidekiq
   module Sqs
     module Fetcher
+      QueueNotDoneError = Class.new(StandardError)
+
       extend ActiveSupport::Concern
 
       included do
@@ -12,7 +14,7 @@ module Sidekiq
         initialize_without_sqs(mgr, queues, strict)
         
         # Fix Queue names
-        @queues = @queues.map {|queue| queue.gsub(/^queue:/, '') }
+        @queues = @queues.map {|queue| Sidekiq::Client.environmental_queue_name queue.gsub(/^queue:/, '') }
         @unique_queues = @queues.uniq
       end
 
@@ -29,14 +31,17 @@ module Sidekiq
             queues.pop # Last entry is TIMEOUT
 
             msg = queues.inject(nil) do |message, queue|
-              message || Sidekiq.sqs.queues.named(queue).receive_message
+              message || fetch_single_message(queue)
             end
 
             if msg
               @mgr.assign!(msg, File.basename(msg.queue.url))
             else
-              after(0) { fetch }
+              after(5) { fetch }
             end
+          rescue QueueNotDoneError
+            # Just wait
+            after(5) { fetch }
           rescue => ex
             logger.error("Error fetching message from queues (#{@queues.join(', ')}): #{ex}")
             logger.error(ex.backtrace.first)
@@ -45,6 +50,20 @@ module Sidekiq
           end
         end
       end
+
+      def fetch_single_message(queue)
+        queue = Sidekiq.sqs.queues.named(queue)
+        message = queue.receive_message
+
+        if @strictly_ordered_queues && message.nil?
+          # If messages being processed, raise
+          if queue.invisible_messages != 0
+            raise QueueNotDoneError
+          end
+        else
+          message
+        end
+      end
     end
   end
 end

data/lib/sidekiq-sqs/manager.rb

@@ -39,7 +39,7 @@ module Sidekiq
 
           @done = true
           Sidekiq::Fetcher.done!
-          @fetcher.finalize
+          #@fetcher.finalize
 
           logger.info { "Shutting down #{@ready.size} quiet workers" }
           @ready.each { |x| x.terminate if x.alive? }

data/lib/sidekiq-sqs/processor.rb

@@ -7,17 +7,29 @@ module Sidekiq
       extend ActiveSupport::Concern
 
       included do
+        remove_method :stats
         alias_method_chain :process, :sqs
       end
 
       def process_with_sqs(sqs_message, queue)
         begin
-          process_without_sqs(Zlib::Inflate.inflate(Base64.decode64(sqs_message.body)), queue)
-        ensure
+          process_without_sqs(Zlib::Inflate.inflate(Base64.decode64(sqs_message.body)), queue).tap do
+            sqs_message.delete
+          end
+        rescue Celluloid::Task::TerminatedError => error
+          # If our thread was killed, requeue the job (SURE HOPE IT'S IDEMPOTENT!)
+          sqs_message.visibility_timeout = 10 
+        rescue => fatal
           # FIXME Maybe we want to requeue here, if there's a non-job related error?
+          # If retry = true, requeue here
           sqs_message.delete
         end
       end
+
+      def stats(*args)
+        # FIXME this should, like, do stuff
+        yield
+      end
     end
   end
 end

data/lib/sidekiq-sqs/util.rb

@@ -1,7 +1,22 @@
+require 'active_support/inflector'
+
 module Sidekiq
   module Util
     def constantize(string)
-      string.constantize
+      # Try just requiring the files first
+      
+      begin
+        string.constantize
+      rescue LoadError => ex
+        begin
+          require string.split("::").map(&:underscore).join("/")
+          string.constantize
+        rescue LoadError => ex
+          STDERR.puts("Error loading constant: #{string}. #{ex.message}")
+          STDERR.puts(ex.backtrace.join("\n"))
+          raise ex
+        end
+      end
     end
   end
 end

data/lib/sidekiq-sqs/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Sqs
-    VERSION = "0.0.11"
+    VERSION = "0.0.17"
   end
 end

data/spec/sidekiq-sqs/client_spec.rb

@@ -28,6 +28,65 @@ describe Sidekiq::Sqs::Client do
     end
   end
 
+  describe ".environmental_queue_name" do
+    before do
+      if defined?(::Rails)
+        ::Rails.stubs(env: "test")
+      else
+        ::Rails = stub(env: "test")
+      end
+    end
+
+    it "prepends the environment name" do
+      subject.environmental_queue_name("queue").should eq("test_queue")
+    end
+
+    it "doesn't prepend if it's already prepended" do
+      subject.environmental_queue_name("test_queue").should eq("test_queue")
+    end
+  end
+
+  describe ".push" do
+
+    context "when scheduling a job" do
+      let(:item) { { 'class' => :klass } }
+
+      it "dispatches to .send_message with delay_seconds" do
+        normalized = { 'at' => Time.now.to_f + 900, 'jid' => 'abc123' }
+        subject.expects(:normalize_item).with(item).returns(normalized)
+        subject.expects(:process_single).with(:klass, normalized).returns([normalized, :payload])
+
+        queue = mock { expects(:send_message).with(:payload, { delay_seconds: 900 }).returns(true) }
+        subject.stubs(queue_or_create: queue)
+
+        subject.push(item).should eq('abc123')
+      end
+
+      it "raises an error if the amount to delay is > 900 seconds" do
+        normalized = { 'at' => Time.now.to_f + 901 }
+        subject.expects(:normalize_item).with(item).returns(normalized)
+        subject.expects(:process_single).with(:klass, normalized).returns([normalized, :payload])
+        subject.stubs(:queue_or_create)
+
+        expect { subject.push(item) }.to raise_error
+      end
+
+    end
+
+    context "when queueing a job normally" do
+      let(:item) { { 'class' => :klass } }
+      let(:queue) { mock { expects(:send_message).with(:payload) } }
+
+      it "dispatches to .send_message" do
+        subject.expects(:normalize_item).with(item).returns(:normalized)
+        subject.expects(:process_single).with(:klass, :normalized).returns([:normalized, :payload])
+        subject.stubs(queue_or_create: queue)
+
+        subject.push(item)
+      end
+    end
+  end
+
   describe ".bulk_send_to_sqs" do
     let(:retryable) do
       {:error_code => 'ServiceUnavailable', :message_body => "blarg"}
@@ -132,12 +191,14 @@ describe Sidekiq::Sqs::Client do
     before { Sidekiq.stubs(sqs: stub(queues: queues)) }
 
     it "returns the queue if it exists" do
+      subject.expects(:environmental_queue_name).with(queue).returns(queue)
       queues.expects(:named).with(queue).returns(:queue)
 
       subject.queue_or_create(queue).should eq(:queue)
     end
 
     it "creates the queue if it doesn't exists" do
+      subject.expects(:environmental_queue_name).with(queue).returns(queue)
       queues.expects(:named).with(queue).raises(AWS::SQS::Errors::NonExistentQueue.new)
       queues.expects(:create).with(queue).returns(:queue)
 

data/spec/sidekiq-sqs/fetcher_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+class FetcherStub
+  def fetch
+  end
+
+  def initialize(mgr, queues, strict)
+    @mgr, @queues, @strictly_ordered_queues = mgr, queues, strict
+  end
+
+  include Sidekiq::Sqs::Fetcher
+end
+
+describe Sidekiq::Sqs::Fetcher do
+  let(:manager) { stub }
+  let(:queues) { [] }
+  let(:queue) { stub }
+
+  before { Sidekiq.stubs(sqs: stub(queues: stub(named: queue))) }
+
+  subject { FetcherStub.new(manager, queues, strict) }
+  context "in strict mode" do
+    let(:strict) { true }
+
+    describe "#fetch_single_message" do
+      it "returns a message if found" do
+        queue.stubs(receive_message: :message)
+
+        subject.fetch_single_message(:queue).should eq(:message)
+      end
+
+      it "returns nil if no messages found when not processing" do
+        queue.stubs(receive_message: nil, invisible_messages: 0)
+
+        subject.fetch_single_message(:queue).should be_nil
+      end
+
+      it "raises an error if no messages found when processing" do
+        queue.stubs(receive_message: nil, invisible_messages: 1)
+
+        expect do
+          subject.fetch_single_message(:queue)
+        end.to raise_error(Sidekiq::Sqs::Fetcher::QueueNotDoneError)
+      end
+    end
+  end
+
+  context "in not strict mode" do
+    let(:strict) { false }
+    describe "#fetch_single_message" do
+      it "returns nil if no messages and processing" do
+        queue.stubs(receive_message: nil, invisible_messages: 1)
+
+        subject.fetch_single_message(:queue).should be_nil
+      end
+    end
+  end
+end

data/spec/sidekiq-sqs/processor_spec.rb

@@ -4,6 +4,10 @@ class StubProcessor
   def process(msg, queue)
   end
 
+  def stats(*args)
+    raise RuntimeError.new
+  end
+
   include Sidekiq::Sqs::Processor
 end
 
@@ -22,4 +26,10 @@ describe Sidekiq::Sqs::Processor do
       subject.process(message, queue).should eq(:processed)
     end
   end
+
+  describe "#stats" do
+    it "just yields" do
+      expect {|b| subject.stats(&b) }.to yield_control
+    end
+  end
 end

metadata

@@ -1,152 +1,115 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: sidekiq-sqs
-version: !ruby/object:Gem::Version
-  version: 0.0.11
+version: !ruby/object:Gem::Version 
   prerelease: 
+  version: 0.0.17
 platform: ruby
-authors:
+authors: 
 - Jon Moses
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-19 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2013-02-04 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>'
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-  type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>'
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
         version: 3.0.0
-- !ruby/object:Gem::Dependency
-  name: sidekiq
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 2.3.2
   type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: sidekiq
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 2.3.2
-- !ruby/object:Gem::Dependency
-  name: aws-sdk
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.6.6
   type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: aws-sdk
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 1.6.6
-- !ruby/object:Gem::Dependency
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
   name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rb-fsevent
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.9.1
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
   type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rb-fsevent
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 0.9.1
-- !ruby/object:Gem::Dependency
-  name: guard
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: guard
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: guard-rspec
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
   type: :development
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: guard-rspec
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: mocha
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.12.3
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
   type: :development
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: mocha
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 0.12.3
-description: SQS backed job store for Sidekiq. Redis is still used for stats/job worker
-  tracking
-email:
+  type: :development
+  version_requirements: *id008
+description: SQS backed job store for Sidekiq. Redis is still used for stats/job worker tracking
+email: 
 - jon@burningbush.us
 executables: []
+
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - .gitignore
 - Gemfile
 - Guardfile
@@ -165,36 +128,41 @@ files:
 - lib/top_level.rb
 - sidekiq-sqs.gemspec
 - spec/sidekiq-sqs/client_spec.rb
+- spec/sidekiq-sqs/fetcher_spec.rb
 - spec/sidekiq-sqs/processor_spec.rb
 - spec/sidekiq-sqs/worker_spec.rb
 - spec/spec_helper.rb
 - spec/util_spec.rb
 homepage: http://github.com/jmoses/sidekiq-sqs
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
 requirements: []
+
 rubyforge_project: 
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: SQS backed job store for Sidekiq
-test_files:
+test_files: 
 - spec/sidekiq-sqs/client_spec.rb
+- spec/sidekiq-sqs/fetcher_spec.rb
 - spec/sidekiq-sqs/processor_spec.rb
 - spec/sidekiq-sqs/worker_spec.rb
 - spec/spec_helper.rb