sidekiq-sqs 0.0.6 → 0.0.7

data/lib/sidekiq-sqs.rb

@@ -1,4 +1,5 @@
 require 'active_support'
+require 'aws-sdk'
 
 require "sidekiq-sqs/version"
 require 'sidekiq-sqs/manager'
@@ -27,7 +28,7 @@ module Sidekiq
 
       # Can't figure how to include/extend and not get a private method...
       def Sidekiq.sqs
-        @sqs_connection ||= AWS::SQS.new
+        AWS::SQS.new
       end
 
       Sidekiq.configure_server do |config|

data/lib/sidekiq-sqs/fetcher.rb

@@ -16,6 +16,16 @@ module Sidekiq
         @unique_queues = @queues.uniq
       end
 
+      # TODO Since there's only one fetcher per manager, we run into the issue
+      #      where it takes longer to fetch a single job that it does to process,
+      #      on average, so that we have waiting workers even if we have jobs in the
+      #      queue because, with the HTTP round trip, fetching single messages is too
+      #      slow.
+      #
+      #      We could fetch 10 at a time, but then we have to worry about stuffing them
+      #      into a "cache", and pushing them back into SQS if we die or exit. We could,
+      #      I guess, just let the "hold" on them expire, and then they would be picked
+      #      up again.  I wonder if that would be 'good enough'
       def fetch
         watchdog('Fetcher#fetch died') do
           return if Sidekiq::Fetcher.done?

data/lib/sidekiq-sqs/manager.rb

@@ -4,7 +4,14 @@ module Sidekiq
       extend ActiveSupport::Concern
 
       included do
-        remove_method :assign
+        remove_method :assign, :dispatch, :stop
+        alias_method_chain :initialize, :sqs
+      end
+
+      def initialize_with_sqs(options = {})
+        initialize_without_sqs(options)
+
+        @fetcher = Sidekiq::Fetcher.pool(args: [current_actor, options[:queues], !!options[:strict]])
       end
 
       def assign(msg, queue)
@@ -24,6 +31,44 @@ module Sidekiq
           end
         end
       end
+
+      def stop(options={})
+        watchdog('Manager#stop died') do
+          shutdown = options[:shutdown]
+          timeout = options[:timeout]
+
+          @done = true
+          Sidekiq::Fetcher.done!
+          @fetcher.finalize
+
+          logger.info { "Shutting down #{@ready.size} quiet workers" }
+          @ready.each { |x| x.terminate if x.alive? }
+          @ready.clear
+
+          logger.debug { "Clearing workers in redis" }
+          Sidekiq.redis do |conn|
+            workers = conn.smembers('workers')
+            workers.each do |name|
+              conn.srem('workers', name) if name =~ /:#{process_id}-/
+            end
+          end
+
+          return after(0) { signal(:shutdown) } if @busy.empty?
+          logger.info { "Pausing up to #{timeout} seconds to allow workers to finish..." }
+          hard_shutdown_in timeout if shutdown
+        end
+      end
+
+      private
+      def dispatch
+        return if stopped?
+        # This is a safety check to ensure we haven't leaked
+        # processors somehow.
+        raise "BUG: No processors, cannot continue!" if @ready.empty? && @busy.empty?
+        raise "No ready processor!?" if @ready.empty?
+
+        @fetcher.fetch!
+      end
     end
   end
 end

data/lib/sidekiq-sqs/processor.rb

@@ -14,7 +14,7 @@ module Sidekiq
         begin
           process_without_sqs(Zlib::Inflate.inflate(Base64.decode64(sqs_message.body)), queue)
         ensure
-          # FIXME Maybe we want to requeue here?
+          # FIXME Maybe we want to requeue here, if there's a non-job related error?
           sqs_message.delete
         end
       end

data/lib/sidekiq-sqs/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Sqs
-    VERSION = "0.0.6"
+    VERSION = "0.0.7"
   end
 end

data/sidekiq-sqs.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |gem|
   gem.email         = ["jon@burningbush.us"]
   gem.description   = %q{SQS backed job store for Sidekiq. Redis is still used for stats/job worker tracking}
   gem.summary       = %q{SQS backed job store for Sidekiq}
-  gem.homepage      = ""
+  gem.homepage      = "http://github.com/jmoses/sidekiq-sqs"
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-sqs
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-28 00:00:00.000000000 Z
+date: 2012-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -132,7 +132,7 @@ files:
 - sidekiq-sqs.gemspec
 - spec/sidekiq-sqs/middleware/compression_spec.rb
 - spec/spec_helper.rb
-homepage: ''
+homepage: http://github.com/jmoses/sidekiq-sqs
 licenses: []
 post_install_message: 
 rdoc_options: []