delayed_job_sqs 0.1.0

data/.gitignore

@@ -0,0 +1,11 @@
+*.swp
+*.tmproj
+.DS_Store
+*.esproj/
+.idea/
+/coverage
+/rdoc
+/pkg
+.rvmrc
+.bundle/config
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,10 @@
+source 'http://rubygems.org'
+
+gemspec
+
+#gem 'aws-sdk', '>= 1.12.0'
+#gem 'delayed_job',  '>= 3.0.0'
+
+#group :development do
+#  gem 'rspec', '>= 2.0'
+#end

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013 LeanDog, Inc. and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,86 @@
+This is an [Amazon SQS](http://aws.amazon.com/sqs/) backend for [delayed_job](http://github.com/collectiveidea/delayed_job)
+
+# Getting Started
+
+## Get credentials
+
+To start using delayed_job_sqs, you need to sign up for an AWS account and setup your credentials.
+
+1. Go to https://portal.aws.amazon.com/gp/aws/developer/registration/index.html and sign up.
+2. Get your AWS Access Key Id & Secret Access Key for your account.
+3. Either `require 'aws-sdk'` and call `AWS.config()` prior to `require 'delayed_job_sqs'`,  or create an aws.yml that will get loaded with your app. You will need to configure Delayed::Worker below to pass in the YAML file location:
+
+```yaml
+[aws]
+  access_key_id: <your access key>
+  secret_access_key: <your secret key>
+```
+
+Or if using Rails, create config/initializers/aws-sdk.rb and put the following into the file:
+
+```ruby
+AWS.config({
+  access_key_id: '<your access key>',
+  secret_access_key: '<your secret key>',
+})
+```
+
+## Installation
+
+Add the gems to your `Gemfile:`
+
+```ruby
+gem 'delayed_job'
+gem 'delayed_job_sqs'
+```
+
+Optionally: Add an initializer (`config/initializers/delayed_job.rb`):
+
+```ruby
+Delayed::Worker.configure do |config|
+  # optional params:
+  config.aws_config = '~/stuff/things/aws.yml' # Specify the file location of the AWS configuration YAML if you're not using Rails and you want to use a YAML file instead of calling AWS.config
+  config.default_queue_name = 'default' # Specify an alternative default queue name
+  config.delay_seconds = # Sets the default delay in seconds for messages sent to the queue.
+  config.message_retention_period = 345600 # The number of seconds Amazon SQS retains a message. Must be an integer from 3600 (1 hour) to 1209600 (14 days). The default for this attribute is 345600 (4 days).
+  config.visibility_timeout = 30 # The length of time (in seconds) that a message received from a queue will be invisible to other receiving components when they ask to receive messages. Valid values: integers from 0 to 43200 (12 hours).
+  config.wait_time_seconds = # How many seconds to wait for a response
+end
+```
+
+## Usage
+
+That's it. Use [delayed_job as normal](http://github.com/collectiveidea/delayed_job).
+
+Example:
+
+```ruby
+class User
+  def background_stuff
+    puts "I run in the background"
+  end
+end
+```
+
+Then in one of your controllers:
+
+```ruby
+user = User.new
+user.delay.background_stuff
+```
+
+## Start worker process
+
+    rake jobs:work
+
+If you want to process a specific queue that's not in your initializer or called `default`, use the `QUEUE` or `QUEUES` environment variable:
+
+    QUEUE=tracking rake jobs:work
+
+That will start pulling jobs off the default queue and processing them. The gem will also handle multiple named queues if you have configured `rake` or `scripts/delayed_job` accordingly however be sure to name the queue when putting objects on the queue:
+
+```ruby
+user = User.new
+user.delay(queue: 'bestest_queue').background_stuff
+```
+

data/Rakefile

@@ -0,0 +1,15 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'rake'
+

data/delayed_job_sqs.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'delayed/backend/version'
+
+Gem::Specification.new do |s|
+  s.require_paths = ["lib"]
+  s.name          = "delayed_job_sqs"
+  s.version       = Delayed::Backend::Sqs.version
+  s.authors       = ["Eric Hankinson"]
+  s.email         = ["eric.hankinson@gmail.com"]
+  s.description   = "Amazon SQS backend for delayed_job"
+  s.summary       = "Amazon SQS backend for delayed_job"
+  s.homepage      = "https://github.com/kumichou/delayed_job_sqs"
+  s.license       = "MIT"
+
+  s.files         = `git ls-files`.split($/)
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+
+  s.add_dependency("aws-sdk", "1.11.1")
+  s.add_dependency("delayed_job", ">= 3.0.0")
+
+  s.add_development_dependency "rspec"
+end
+

data/lib/delayed/backend/actions.rb

@@ -0,0 +1,70 @@
+module Delayed
+  module Backend
+    module Sqs
+      module Actions
+        def field(name, options = {})
+          default = options[:default] || nil
+          define_method name do
+            @attributes ||= {}
+            @attributes[name.to_sym] || default
+          end
+
+          define_method "#{name}=" do |value|
+            @attributes ||= {}
+            @attributes[name.to_sym] = value
+          end
+        end
+
+        def before_fork
+        end
+
+        def after_fork
+        end
+
+        def db_time_now
+          Time.now.utc
+        end
+
+        def find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time)
+          Delayed::Worker.queues.each_with_index do |queue, index|
+            message = sqs.queues.named(queue_name(index)).receive_message
+            return [Delayed::Backend::Sqs::Job.new(message)] if message
+          end
+          []
+        end
+
+        def delete_all
+          deleted = 0
+
+          Delayed::Worker.queues.each_with_index do |queue, index|
+            loop do
+              msgs = sqs.queues.named(queue_name(index)).receive_message({ :limit => 10})
+              break if msgs.blank?
+              msgs.each do |msg|
+                msg.delete
+                deleted += 1
+              end
+            end
+          end
+
+          puts "Messages removed: #{deleted}"
+        end
+
+        # No need to check locks
+        def clear_locks!(*args)
+          true
+        end
+
+        private
+
+        def sqs
+          ::Delayed::Worker.sqs
+        end
+
+        def queue_name(index)
+          Delayed::Worker.queues[index]
+        end
+      end
+    end
+  end
+end

data/lib/delayed/backend/sqs.rb

@@ -0,0 +1,120 @@
+
+module Delayed
+  module Backend
+    module Sqs
+      class Job
+        include ::DelayedJobSqs::Document
+        include Delayed::Backend::Base
+        extend  Delayed::Backend::Sqs::Actions
+
+        field :priority,    :type => Integer, :default => 0
+        field :attempts,    :type => Integer, :default => 0
+        field :handler,     :type => String
+        field :run_at,      :type => Time
+        field :locked_at,   :type => Time
+        field :locked_by,   :type => String
+        field :failed_at,   :type => Time
+        field :last_error,  :type => String
+        field :queue,       :type => String
+
+        def initialize(data = {})
+          puts "[init] Delayed::Backend::Sqs"
+          @msg = nil
+
+          if data.is_a?(AWS::SQS::ReceivedMessage)
+            @msg = data
+            data = JSON.load(data.body)
+          end
+
+          data.symbolize_keys!
+          payload_obj = data.delete(:payload_object) || data.delete(:handler)
+
+          @queue_name = data[:queue]      || Delayed::Worker.default_queue_name
+          @delay      = data[:delay]      || Delayed::Worker.delay
+          @timeout    = data[:timeout]    || Delayed::Worker.timeout
+          @expires_in = data[:expires_in] || Delayed::Worker.expires_in
+          @attributes = data
+          self.payload_object = payload_obj
+        end
+
+        def payload_object
+          @payload_object ||= YAML.load(self.handler)
+        rescue TypeError, LoadError, NameError, ArgumentError => e
+          raise DeserializationError,
+            "Job failed to load: #{e.message}. Handler: #{handler.inspect}"
+        end
+
+        def payload_object=(object)
+          if object.is_a? String
+            @payload_object = YAML.load(object)
+            self.handler = object
+          else
+            @payload_object = object
+            self.handler = object.to_yaml
+          end
+        end
+
+        def save
+          puts "[SAVE] #{@attributes.inspect}"
+
+          if @attributes[:handler].blank?
+            raise "Handler missing!"
+          end
+          payload = JSON.dump(@attributes)
+
+          @msg.delete if @msg
+
+          sqs.queues.named(queue_name).send_message(payload, :delay_seconds  => @delay)
+          true
+        end
+
+        def save!
+          save
+        end
+
+        def destroy
+          if @msg
+            puts "job destroyed! #{@msg.id}"
+            @msg.delete
+          end
+        end
+
+        def fail!
+          destroy
+          # v2: move to separate queue
+        end
+
+        def update_attributes(attributes)
+          attributes.symbolize_keys!
+          @attributes.merge attributes
+          save
+        end
+
+        # No need to check locks
+        def lock_exclusively!(*args)
+          true
+        end
+
+        # No need to check locks
+        def unlock(*args)
+          true
+        end
+
+        def reload(*args)
+          # reset
+          super
+        end
+
+        private
+
+        def queue_name
+          @queue_name
+        end
+
+        def sqs
+          ::Delayed::Worker.sqs
+        end
+      end
+    end
+  end
+end

data/lib/delayed/backend/sqs_config.rb

@@ -0,0 +1,3 @@
+class SqsConfig
+    attr_accessor :project_id, :default_queue_name, :delay_seconds, :visibility_timeout, :message_retention_period, :aws_config
+end

data/lib/delayed/backend/version.rb

@@ -0,0 +1,11 @@
+module Delayed
+  module Backend
+    module Sqs
+      @@version = nil
+
+      def self.version
+        @@version ||= "0.1.0"
+      end
+    end
+  end
+end

data/lib/delayed/backend/worker.rb

@@ -0,0 +1,62 @@
+require_relative 'sqs_config'
+
+module Delayed
+  class Worker
+
+    class << self
+      attr_accessor :config, :sqs, :delay, :timeout, :expires_in, :aws_config
+
+      def configure
+        yield(config)
+
+        self.default_queue_name = if !config.default_queue_name.nil? && config.default_queue_name.length != 0
+                        config.default_queue_name
+                      else
+                        'default'
+                      end
+        self.delay = config.delay_seconds || 0
+        self.timeout = config.visibility_timeout || 5.minutes
+        self.expires_in = config.message_retention_period || 4.days
+      end
+
+      def config
+        @config ||= SqsConfig.new
+      end
+    end
+  end
+
+  module Backend
+    module Sqs
+      if Object.const_defined?(:Rails) and Rails.const_defined?(:Railtie)
+        class Railtie < Rails::Railtie
+
+          # configure our gem after Rails completely boots so that we have
+          # access to any config/initializers that were run
+          config.after_initialize do
+            AWS::Rails.setup
+
+            Delayed::Worker.sqs = AWS::SQS.new
+            Delayed::Worker.configure {}
+          end
+        end
+      else
+        path = Pathname.new(Delayed::Worker.config.aws_config)
+
+        if File.exists?(path)
+          cfg = YAML::load(File.read(path))
+
+          unless cfg.keys[0]
+            raise "AWS Yaml configuration file is missing a section"
+          end
+
+          AWS.config(cfg.keys[0])
+        end
+
+        Delayed::Worker.sqs = AWS::SQS.new
+        Delayed::Worker.configure {}
+      end
+    end
+  end
+end
+
+

data/lib/delayed/serialization/sqs.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+
+module DelayedJobSqs
+  module Document
+  end
+end

data/lib/delayed_job_sqs.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+require 'aws-sdk'
+require 'delayed_job'
+require 'active_support'
+
+module ActiveSupport
+  Inflector.inflections do |inflect|
+    inflect.irregular('sqs', 'sqs')
+  end
+end
+
+require_relative 'delayed/serialization/sqs'
+require_relative 'delayed/backend/actions'
+require_relative 'delayed/backend/sqs_config'
+require_relative 'delayed/backend/worker'
+require_relative 'delayed/backend/version'
+require_relative 'delayed/backend/sqs'
+
+Delayed::Worker.backend = :sqs
+

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: delayed_job_sqs
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Eric Hankinson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.11.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.11.1
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Amazon SQS backend for delayed_job
+email:
+- eric.hankinson@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- delayed_job_sqs.gemspec
+- lib/delayed/backend/actions.rb
+- lib/delayed/backend/sqs.rb
+- lib/delayed/backend/sqs_config.rb
+- lib/delayed/backend/version.rb
+- lib/delayed/backend/worker.rb
+- lib/delayed/serialization/sqs.rb
+- lib/delayed_job_sqs.rb
+- spec/.gitkeep
+homepage: https://github.com/kumichou/delayed_job_sqs
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Amazon SQS backend for delayed_job
+test_files:
+- spec/.gitkeep