sidekiq-sqs 0.0.8 → 0.0.9

data/lib/sidekiq-sqs.rb

@@ -7,6 +7,7 @@ require 'sidekiq-sqs/fetcher'
 require 'sidekiq-sqs/client'
 require 'sidekiq-sqs/processor'
 require 'sidekiq-sqs/worker'
+require 'sidekiq-sqs/aws-sdk/batch_send_failure_patch'
 
 # TODO The retry server middleware directly writes to a retry zset.
 # TODO Need a queue-prefix option to support multiple rails envs
@@ -25,6 +26,7 @@ module Sidekiq
       Sidekiq::Client.send :include, Sidekiq::Sqs::Client
       Sidekiq::Processor.send :include, Sidekiq::Sqs::Processor
       Sidekiq::Worker::ClassMethods.send :include, Sidekiq::Sqs::Worker
+      AWS::SQS::Queue.send :include, Sidekiq::Sqs::AwsSdk::BatchSendFailurePatch
 
       # Can't figure how to include/extend and not get a private method...
       def Sidekiq.sqs

data/lib/sidekiq-sqs/aws-sdk/batch_send_failure_patch.rb

@@ -0,0 +1,34 @@
+module Sidekiq
+  module Sqs
+    module AwsSdk
+      module BatchSendFailurePatch
+        extend ActiveSupport::Concern
+
+        included do
+          remove_method :batch_failures
+        end
+
+        def batch_failures entries, response
+          response[:failed].inject([]) do |failures, failure|
+
+            entry = entries.find{|e| e[:id] == failure[:id] }
+
+            details = {
+              :error_code => failure[:code],
+              :error_message => failure[:message],
+              :sender_fault => failure[:sender_fault],
+              :message => entry[:message_body] # We need this for retrying
+            }
+
+            if handle = entry[:receipt_handle]
+              details[:receipt_handle] = handle
+            end
+
+            failures << details
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/sidekiq-sqs/client.rb

@@ -6,6 +6,25 @@ module Sidekiq
     module Client
       extend ActiveSupport::Concern
 
+      RETRYABLE_ERRORS = %w(
+        AWS.SimpleQueueService.InternalError
+        InternalError
+        RequestThrottled
+        ServiceUnavailable
+      )
+
+      class BulkInsertionError < StandardError
+        attr_reader :failed
+
+        def initialize(message, failed = [])
+          super(message)
+          @failed = failed
+        end
+      end
+
+      class Retryable < StandardError
+      end
+
       included do
         class << self
           remove_method :push
@@ -31,22 +50,71 @@ module Sidekiq
           pushed ? normed['jid'] : nil
         end
 
+        MAX_BULK_RETRIES = 5
         def push_bulk(items)
+          queue_name, payloads = format_items(items)
+          queue = queue_or_create(queue_name)
+
+          failures, can_retry = bulk_send_to_sqs(queue, payloads)
+
+          retries = 0
+          begin
+            if can_retry.size > 0
+              failed, can_retry = bulk_send_to_sqs(queue, can_retry)
+              failures.concat failed
+
+              raise Retryable if can_retry.size > 0
+            end
+          rescue Retryable
+            sleep retries ** 2
+            retry if (retries += 1) < MAX_BULK_RETRIES
+          end
+
+          if failures.size > 0
+            raise BulkInsertionError.new("Some messages failed to insert", failed)
+          end
+
+          failures.empty? ? payloads.size : nil
+        end
+
+        def format_items(items)
           normed = normalize_item(items)
           payloads = items['args'].map do |args|
             _, payload = process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12)))
             payload
           end.compact
 
-          pushed = false
-          queue = queue_or_create(normed['queue'])
+          [normed['queue'], payloads]
+        end
 
-          payloads.each_slice(10) do |items|
-            ## FIXME error handling
-            pushed = queue.batch_send( items ) 
+        def bulk_send_to_sqs(queue, formatted_items)
+          failures = []
+          can_retry = []
+          formatted_items.each_slice(10) do |items|
+            failed, retryable = send_batch_to_sqs(queue, items)
+
+            failures.concat failed
+            can_retry.concat retryable
           end
 
-          pushed ? payloads.size : nil
+          [failures, can_retry]
+        end
+
+        def send_batch_to_sqs(queue, formatted_items)
+          failures, retryables = [], []
+          
+          begin
+            queue.batch_send(formatted_items)
+          rescue AWS::SQS::Errors::BatchSendError => error
+            retryable, failed = error.failures.partition do |failure|
+              RETRYABLE_ERRORS.include?(failure[:error_code])
+            end
+
+            failures.concat failed
+            retryables.concat retryable
+          end
+          
+          [failures, retryables]
         end
 
         def queue_or_create(queue)

data/lib/sidekiq-sqs/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Sqs
-    VERSION = "0.0.8"
+    VERSION = "0.0.9"
   end
 end

data/spec/sidekiq-sqs/client_spec.rb

@@ -10,6 +10,10 @@ class StubClient
   def self.process_single(*args)
   end
 
+  def self.normalize_item(*args)
+    Hash.new
+  end
+
   include Sidekiq::Sqs::Client
 end
 
@@ -24,6 +28,104 @@ describe Sidekiq::Sqs::Client do
     end
   end
 
+  describe ".bulk_send_to_sqs" do
+    let(:retryable) do
+      {:error_code => 'ServiceUnavailable', :message_body => "blarg"}
+    end
+    let(:failed) do
+      {:error_code => "GFYS", :message_body => "and your little dog, too"}
+    end
+    it "dispatches to .send_batch_to_sqs in groups of 10" do
+      items = 1.upto(20).to_a
+      subject.expects(:send_batch_to_sqs).with(:queue, 1.upto(10).to_a).returns([[], []])
+      subject.expects(:send_batch_to_sqs).with(:queue, 11.upto(20).to_a).returns([[], []])
+
+      subject.bulk_send_to_sqs(:queue, items)
+    end
+
+    it "aggregates failed and retryable messages" do
+      items = 1.upto(20).to_a
+      subject.expects(:send_batch_to_sqs).with(:queue, 1.upto(10).to_a).returns([[failed], []])
+      subject.expects(:send_batch_to_sqs).with(:queue, 11.upto(20).to_a).returns([[], [retryable]])
+
+      subject.bulk_send_to_sqs(:queue, items).should eq([[failed], [retryable]])
+    end
+  end
+
+  describe ".send_batch_to_sqs" do
+    let(:queue) { stub }
+    let(:retryable) do
+      {:error_code => 'ServiceUnavailable', :message_body => "blarg"}
+    end
+    let(:failed) do
+      {:error_code => "GFYS", :message_body => "and your little dog, too"}
+    end
+
+    it "dispatches to the queue" do
+      queue.expects(:batch_send).with(:items)
+
+      subject.send_batch_to_sqs(queue, :items).should eq([[], []])
+    end
+
+    it "aggregates errors correctly" do
+      queue.expects(:batch_send).with(:items).raises(
+        AWS::SQS::Errors::BatchSendError.new(
+          [:sent],
+          [failed, retryable]
+        )
+      )
+
+      subject.send_batch_to_sqs(queue, :items).should eq([
+        [failed], [retryable]
+      ])
+    end
+  end
+
+  describe ".push_bulk" do
+    let(:queue) { stub }
+    let(:retries) { [] }
+    let(:fails) { [] }
+
+    before do
+      subject.stubs(queue_or_create: queue)
+    end
+
+
+    it "needs more tests"
+
+    context "when some messages fail to insert" do
+      before do
+        subject.expects(:bulk_send_to_sqs).with(queue, :payloads).returns([fails, retries])
+      end
+
+      context "and all are retryable" do
+        before do
+          retries.push :error
+
+          subject.expects(:format_items).with(:items).returns([:queue, :payloads])
+        end
+
+        it "retries" do
+          subject.expects(:bulk_send_to_sqs).with(queue, [:error]).returns([[], []])
+
+          subject.push_bulk(:items)
+        end
+      end
+
+      context "and there are non-tryable failures" do
+        before do
+          fails.push :fail
+
+          subject.expects(:format_items).with(:items).returns([:queue, :payloads])
+        end
+
+        it "raises an error" do
+          expect { subject.push_bulk :items }.to raise_error(Sidekiq::Sqs::Client::BulkInsertionError)
+        end
+      end
+    end
+  end
+
   describe ".queue_or_create" do
     let(:queue) { 'queue' }
     let(:queues) { stub }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-sqs
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-09 00:00:00.000000000 Z
+date: 2012-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -154,6 +154,7 @@ files:
 - README.md
 - Rakefile
 - lib/sidekiq-sqs.rb
+- lib/sidekiq-sqs/aws-sdk/batch_send_failure_patch.rb
 - lib/sidekiq-sqs/client.rb
 - lib/sidekiq-sqs/fetcher.rb
 - lib/sidekiq-sqs/manager.rb