sidekiq-sqs 0.0.6a

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sidekiq-sqs.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,24 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  # Rails example
+  watch(%r{^app/(.+)\.rb$})                           { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^app/(.*)(\.erb|\.haml)$})                 { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+  watch(%r{^app/controllers/(.+)_(controller)\.rb$})  { |m| ["spec/routing/#{m[1]}_routing_spec.rb", "spec/#{m[2]}s/#{m[1]}_#{m[2]}_spec.rb", "spec/acceptance/#{m[1]}_spec.rb"] }
+  watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }
+  watch('config/routes.rb')                           { "spec/routing" }
+  watch('app/controllers/application_controller.rb')  { "spec/controllers" }
+  
+  # Capybara request specs
+  watch(%r{^app/views/(.+)/.*\.(erb|haml)$})          { |m| "spec/requests/#{m[1]}_spec.rb" }
+  
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$})   { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance' }
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Jon Moses
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# Sidekiq::Sqs
+
+This is a gem that modifies Sidekiq to use Amazon's SQS as a job store.  It
+still requires Redis for stats and such, and worker tracking.
+
+It works currently. There's zero tests, and we're not using it in production, so
+you might want to be careful.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sidekiq-sqs'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-sqs
+
+## Usage
+
+Once the gem is required, (and aws-sdk is configured) simply do:
+
+    Sidekiq::Sqs.enable!
+
+That'll turn it all on. Nothing else _should_ have to change.
+
+## Caveats
+
+* Scheduling is completely broken
+* ?
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/sidekiq-sqs.rb

@@ -0,0 +1,42 @@
+require 'active_support'
+
+require "sidekiq-sqs/version"
+require 'sidekiq-sqs/manager'
+require 'sidekiq-sqs/fetcher'
+require 'sidekiq-sqs/client'
+require 'sidekiq-sqs/processor'
+require 'sidekiq-sqs/worker'
+
+# TODO The retry server middleware directly writes to a retry zset.
+# TODO Need a queue-prefix option to support multiple rails envs
+
+module Sidekiq
+  module Sqs
+    module_function
+    def enable!
+      require 'sidekiq/manager'
+      require 'sidekiq/fetch'
+      require 'sidekiq/client'
+      require 'sidekiq/processor'
+
+      Sidekiq::Manager.send :include, Sidekiq::Sqs::Manager
+      Sidekiq::Fetcher.send :include, Sidekiq::Sqs::Fetcher
+      Sidekiq::Client.send :include, Sidekiq::Sqs::Client
+      Sidekiq::Processor.send :include, Sidekiq::Sqs::Processor
+      Sidekiq::Worker::ClassMethods.send :include, Sidekiq::Sqs::Worker
+
+      # Can't figure how to include/extend and not get a private method...
+      def Sidekiq.sqs
+        @sqs_connection ||= AWS::SQS.new
+      end
+
+      Sidekiq.configure_server do |config|
+        config.server_middleware do |chain|
+          chain.remove Sidekiq::Middleware::Server::RetryJobs
+          chain.remove Sidekiq::Middleware::Server::Timeout
+        end
+      end
+    end
+  end
+end
+

data/lib/sidekiq-sqs/client.rb

@@ -0,0 +1,68 @@
+require 'base64'
+require 'zlib'
+
+module Sidekiq
+  module Sqs
+    module Client
+      extend ActiveSupport::Concern
+
+      included do
+        class << self
+          remove_method :push
+          remove_method :push_bulk
+
+          alias_method_chain :process_single, :sqs
+        end
+      end
+
+      module ClassMethods
+        def push(item)
+          normed = normalize_item(item)
+          normed, payload = process_single(item['class'], normed)
+
+          pushed = false
+          if normed['at']
+            # FIXME - SQS only supports a delay of 15.minutes
+            # probably a schedule queue? or keep this in redis??
+            pushed = conn.zadd('schedule', normed['at'].to_s, payload)
+          else
+            pushed = queue_or_create(normed['queue']).send_message(payload)
+          end if normed
+          pushed ? normed['jid'] : nil
+        end
+
+        def push_bulk(items)
+          normed = normalize_item(items)
+          payloads = items['args'].map do |args|
+            _, payload = process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12)))
+            payload
+          end.compact
+
+          pushed = false
+          queue = queue_or_create(normed['queue'])
+
+          payloads.each_slice(10) do |items|
+            ## FIXME error handling
+            pushed = queue.batch_send( items ) 
+          end
+
+          pushed ? payloads.size : nil
+        end
+
+        def queue_or_create(queue)
+          begin
+            Sidekiq.sqs.queues.named(queue.to_s)
+          rescue AWS::SQS::Errors::NonExistentQueue
+            Sidekiq.sqs.queues.create(queue.to_s)
+          end
+        end
+
+        def process_single_with_sqs(worker_class, item)
+          item, payload = process_single_without_sqs(worker_class, item)
+
+          return item, Base64.encode64(Zlib::Deflate.deflate(payload))
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq-sqs/fetcher.rb

@@ -0,0 +1,52 @@
+module Sidekiq
+  module Sqs
+    module Fetcher
+      extend ActiveSupport::Concern
+
+      included do
+        remove_method :fetch
+        alias_method_chain :initialize, :sqs
+      end
+
+      def initialize_with_sqs(mgr, queues, strict)
+        initialize_without_sqs(mgr, queues, strict)
+        
+        # Fix Queue names
+        @queues = @queues.map {|queue| queue.gsub(/^queue:/, '') }
+        @unique_queues = @queues.uniq
+      end
+
+      def fetch
+        watchdog('Fetcher#fetch died') do
+          return if Sidekiq::Fetcher.done?
+
+          begin
+            queue = nil
+            msg = nil
+
+            ## FIXME
+            queues = queues_cmd
+            queues.pop # Last entry is TIMEOUT
+
+            msg = queues.inject(nil) do |message, queue|
+              next if message
+
+              Sidekiq.sqs.queues.named(queue).receive_message
+            end
+
+            if msg
+              @mgr.assign!(msg, File.basename(msg.queue.url))
+            else
+              after(0) { fetch }
+            end
+          rescue => ex
+            logger.error("Error fetching message from queues (#{@queues.join(', ')}): #{ex}")
+            logger.error(ex.backtrace.first)
+            sleep(self.class::TIMEOUT)
+            after(0) { fetch }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq-sqs/manager.rb

@@ -0,0 +1,29 @@
+module Sidekiq
+  module Sqs
+    module Manager
+      extend ActiveSupport::Concern
+
+      included do
+        remove_method :assign
+      end
+
+      def assign(msg, queue)
+        watchdog("Manager#assign died") do
+          if stopped?
+            # Race condition between Manager#stop if Fetcher
+            # is blocked on redis and gets a message after
+            # all the ready Processors have been stopped.
+            # Push the message back to redis.
+            #
+            msg.visibility_timeout = 0
+          else
+            processor = @ready.pop
+            @in_progress[processor.object_id] = [msg, queue]
+            @busy << processor
+            processor.process!(msg, queue)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq-sqs/processor.rb

@@ -0,0 +1,23 @@
+require 'base64'
+require 'zlib'
+
+module Sidekiq
+  module Sqs
+    module Processor
+      extend ActiveSupport::Concern
+
+      included do
+        alias_method_chain :process, :sqs
+      end
+
+      def process_with_sqs(sqs_message, queue)
+        begin
+          process_without_sqs(Zlib::Inflate.inflate(Base64.decode64(sqs_message.body)), queue)
+        ensure
+          # FIXME Maybe we want to requeue here?
+          sqs_message.delete
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq-sqs/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Sqs
+    VERSION = "0.0.6a"
+  end
+end

data/lib/sidekiq-sqs/worker.rb

@@ -0,0 +1,14 @@
+module Sidekiq
+  module Sqs
+    module Worker
+      def perform_async_bulk(args)
+        client_push_bulk('class' => self, 'args' => args)
+      end
+
+      # args should be an array of single job arguments
+      def client_push_bulk(args)
+        Sidekiq::Client.push_bulk(args)
+      end
+    end
+  end
+end

data/lib/top_level.rb

@@ -0,0 +1,9 @@
+module Sidekiq
+  module Sqs
+    module TopLevel
+      def self.sqs
+        @sqs_connection ||= AWS::SQS.new
+      end
+    end
+  end
+end

data/sidekiq-sqs.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq-sqs/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "sidekiq-sqs"
+  gem.version       = Sidekiq::Sqs::VERSION
+  gem.authors       = ["Jon Moses"]
+  gem.email         = ["jon@burningbush.us"]
+  gem.description   = %q{SQS backed job store for Sidekiq. Redis is still used for stats/job worker tracking}
+  gem.summary       = %q{SQS backed job store for Sidekiq}
+  gem.homepage      = "http://github.com/jmoses/sidekiq-sqs"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'activesupport', '> 3.0.0'
+  gem.add_dependency 'sidekiq', '~> 2.3.2'
+  gem.add_dependency 'aws-sdk', '~> 1.6.6'
+
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'guard'
+  gem.add_development_dependency 'guard-rspec'
+end

data/spec/sidekiq-sqs/middleware/compression_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Sidekiq::Sqs::Middleware::Compression do
+  context "Server" do
+    subject { Sidekiq::Sqs::Middleware::Compression::Server.new }
+    let(:message) { stub(body: "eJzLTS0uTkxPBQALwQLm\n") }
+
+    it "decompresses the message" do
+      subject.call(nil, message, nil)
+
+      message.body.should eq('message')
+    end
+  end
+
+#  context "Client" do
+#    subject { Sidekiq::Sqs::Middleware::Compression::Client.new }
+#    let(:message) { 'message' }
+#
+#    it "compresses the message and yields" do
+#      expect {|b| subject.call(nil, message, :queue, &b) }.to yield_control 
+#
+#      message.should eq(Base64.encode64(Zlib::Deflate.deflate('message')))
+#    end
+#  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+$: << 'lib'
+require 'sidekiq-sqs'
+
+

metadata

@@ -0,0 +1,161 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-sqs
+version: !ruby/object:Gem::Version
+  version: 0.0.6a
+  prerelease: 5
+platform: ruby
+authors:
+- Jon Moses
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.6
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: SQS backed job store for Sidekiq. Redis is still used for stats/job worker
+  tracking
+email:
+- jon@burningbush.us
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sidekiq-sqs.rb
+- lib/sidekiq-sqs/client.rb
+- lib/sidekiq-sqs/fetcher.rb
+- lib/sidekiq-sqs/manager.rb
+- lib/sidekiq-sqs/processor.rb
+- lib/sidekiq-sqs/version.rb
+- lib/sidekiq-sqs/worker.rb
+- lib/top_level.rb
+- sidekiq-sqs.gemspec
+- spec/sidekiq-sqs/middleware/compression_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/jmoses/sidekiq-sqs
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: SQS backed job store for Sidekiq
+test_files:
+- spec/sidekiq-sqs/middleware/compression_spec.rb
+- spec/spec_helper.rb