boss_queue 0.1.0

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gem 'aws-sdk'
+
+group :development do
+  gem 'rspec'
+  gem 'bundler'
+  gem 'jeweler', :git => 'https://github.com/technicalpickles/jeweler.git', :branch => :master
+  gem 'pry'
+  gem 'pry-nav'
+  gem 'pry-stack_explorer'
+end

data/Gemfile.lock

@@ -0,0 +1,92 @@
+GIT
+  remote: https://github.com/technicalpickles/jeweler.git
+  revision: f7e0a55a207d83f56637dd8fbabf26a803410faf
+  branch: master
+  specs:
+    jeweler (1.8.7)
+      builder
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      github_api (= 0.10.1)
+      highline (>= 1.6.15)
+      nokogiri (= 1.5.10)
+      rake
+      rdoc
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.5)
+    aws-sdk (1.19.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4, < 1.6.0)
+      uuidtools (~> 2.1)
+    binding_of_caller (0.7.2)
+      debug_inspector (>= 0.0.1)
+    builder (3.2.2)
+    coderay (1.0.9)
+    debug_inspector (0.0.2)
+    diff-lcs (1.2.4)
+    faraday (0.8.8)
+      multipart-post (~> 1.2.0)
+    git (1.2.6)
+    github_api (0.10.1)
+      addressable
+      faraday (~> 0.8.1)
+      hashie (>= 1.2)
+      multi_json (~> 1.4)
+      nokogiri (~> 1.5.2)
+      oauth2
+    hashie (2.0.5)
+    highline (1.6.19)
+    httpauth (0.2.0)
+    json (1.8.0)
+    jwt (0.1.8)
+      multi_json (>= 1.5)
+    method_source (0.8.2)
+    multi_json (1.8.0)
+    multi_xml (0.5.5)
+    multipart-post (1.2.0)
+    nokogiri (1.5.10)
+    oauth2 (0.9.2)
+      faraday (~> 0.8)
+      httpauth (~> 0.2)
+      jwt (~> 0.1.4)
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    pry (0.9.12.2)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    pry-nav (0.2.3)
+      pry (~> 0.9.10)
+    pry-stack_explorer (0.4.9.1)
+      binding_of_caller (>= 0.7)
+      pry (>= 0.9.11)
+    rack (1.5.2)
+    rake (10.1.0)
+    rdoc (4.0.1)
+      json (~> 1.4)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.5)
+    rspec-expectations (2.14.3)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.3)
+    slop (3.4.6)
+    uuidtools (2.1.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-sdk
+  bundler
+  jeweler!
+  pry
+  pry-nav
+  pry-stack_explorer
+  rspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Populr.me
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,81 @@
+boss_queue
+==========
+
+A fault tolerant job queue built around Amazon SQS & DynamoDB
+
+
+Setup
+============
+
+In your Gemfile:
+
+    gem 'boss_queue'
+
+
+boss_queue uses an Amazon SQS queue and a Amazon DynamoDB table for each environment (production, staging, test). To set these up in Rails do:
+
+    $ rails c
+
+    AWS.config(:access_key_id => <access_key_id>,
+               :secret_access_key => <secret_access_key>)
+
+    BossQueue.environment = 'development'
+    BossQueue.create_table
+    BossQueue.create_queue
+
+    BossQueue.environment = 'staging'
+    BossQueue.create_table
+    BossQueue.create_queue
+
+    # BossQueue.create_table(read_capacity, write_capacity)
+    # One read capacity unit = two eventually consistent reads per second, for items up 4 KB in size.
+    # One write capacity unit = one write per second, for items up to 1 KB in size.
+
+    BossQueue.environment = 'production'
+    BossQueue.create_table(50, 10)
+    BossQueue.create_queue
+
+
+Alternatively, in each of the respective environments, do:
+
+    $ rails c
+
+    AWS.config(:access_key_id => <access_key_id>,
+               :secret_access_key => <secret_access_key>)
+
+    # environment does not need to be set because it is taken from Rails.env
+    BossQueue.create_table
+    BossQueue.create_queue
+
+
+Or these could be put into a migration.
+
+
+Usage
+=====
+
+    myobject = MyClass.new
+    BossQueue.failure_action = 'none' # default is 'retry' which retries up to four times
+
+    # can enqueue instance methods (assumes that objects have an id and a #find(id) method)
+    BossQueue.enqueue(myobject, :method_to_execute, arg1, arg2)
+    # enqueue with a delay of up to 900 seconds (15 minutes)
+    BossQueue.enqueue_with_delay(60, myobject, :method_to_execute, arg1, arg2, arg3)
+
+    # can enqueue class methods
+    BossQueue.enqueue(MyClass, :method_to_execute)
+    BossQueue.enqueue_with_delay(60, MyClass, :method_to_execute, arg1, arg2)
+
+    BossQueue.work
+
+    # failures are left in the DynamoDB table with the failed boolean set to true
+
+BossQueue does not at present have a daemon component such as Sidekiq or Resque.
+
+
+Future Work
+===========
+
+Create some mechanism for viewing failed jobs (and perhaps queued jobs...they are all in the same table)
+
+

data/Rakefile

@@ -0,0 +1,49 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "boss_queue"
+  gem.homepage = "https://github.com/populr/boss_queue"
+  gem.license = "MIT"
+  gem.summary = %Q{A fault tolerant job queue built around Amazon SQS & DynamoDB}
+  gem.description = %Q{A fault tolerant job queue built around Amazon SQS & DynamoDB}
+  gem.email = "daniel@populr.me"
+  gem.authors = ["Daniel Nelson"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "boss_queue #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/boss_queue/boss_queue.rb

@@ -0,0 +1,104 @@
+require 'json'
+
+class BossQueue
+  @@environment
+
+  def self.environment=(env)
+    @@environment = env
+  end
+
+  @@failure_action
+
+  def self.failure_action
+    @@failure_action ||= 'retry'
+  end
+
+  def self.failure_action=(env)
+    @@failure_action = env
+  end
+
+
+  def self.table_name
+    "#{self.queue_prefix}boss_queue_jobs"
+  end
+
+  def self.queue_name
+    "#{self.queue_prefix}boss_queue"
+  end
+
+
+  def self.create_table(read_capacity=1, write_capacity=1, options={})
+    create_opts = {}
+    create_opts[:hash_key] = { hash_key => :string }
+    create_opts[:range_key] = { :kind => :string }
+
+    AWS::DynamoDB.new.tables.create(self.table_name, read_capacity, write_capacity, create_opts)
+  end
+
+  def self.create_queue
+    AWS::SQS::QueueCollection.new.create(self.queue_name, :default_visibility_timeout => 5 * 60)
+  end
+
+  def self.work
+    queue = AWS::SQS.new.queues[self.queue_name]
+    queue.receive_message do |job_id|
+      # When a block is given, each message is yielded to the block and then deleted as long as the block exits normally - http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html
+      job = BossQueue::Job.shard(table_name).find_by_id(job_id.body)
+      job.queue_name = self.queue_name
+      job.work
+    end
+  end
+
+  def self.enqueue(class_or_instance, method_name, *args)
+    job = self.create_job(class_or_instance, method_name, *args)
+    job.enqueue
+  end
+
+  def self.enqueue_with_delay(delay, class_or_instance, method_name, *args)
+    job = self.create_job(class_or_instance, method_name, *args)
+    job.enqueue_with_delay(delay)
+  end
+
+  def self.create_job(class_or_instance, method_name, *args) # :nodoc:
+    job = BossQueue::Job.shard(table_name).new
+    if class_or_instance.is_a?(Class)
+      class_name = class_or_instance.to_s
+      instance_id = nil
+      job.kind = "#{class_name}@#{method_name}"
+    else
+      class_name = class_or_instance.class.to_s
+      instance_id = class_or_instance.id
+      job.kind = "#{class_name}##{method_name}"
+    end
+    job.queue_name = self.queue_name
+    job.failure_action = self.failure_action
+    job.model_class_name = class_name
+    job.model_id = instance_id unless instance_id.nil?
+    job.job_method = method_name.to_s
+    job.job_arguments = JSON.generate(args)
+    job.save!
+    job
+  end
+
+  def self.environment # :nodoc:
+    @@environment ||= if Module.const_get('Rails')
+                        Rails.env
+                      elsif Module.const_get('Rack')
+                        Rack.env
+                      else
+                        raise 'BossQueue requires an environment'
+                      end
+  end
+
+  def self.queue_prefix # :nodoc:
+    case self.environment
+    when 'production'
+      ''
+    when 'development'
+      'dev_'
+    else
+      environment + '_'
+    end
+  end
+
+end

data/lib/boss_queue/job.rb

@@ -0,0 +1,100 @@
+require 'json'
+
+class BossQueue
+
+  class Job < AWS::Record::HashModel
+    attr_accessor :queue_name
+
+    string_attr :kind # an index based model_class_name, job_method
+    boolean_attr :failed
+
+    string_attr :model_class_name
+    string_attr :model_id
+    string_attr :job_method
+    string_attr :job_arguments
+
+    integer_attr :failed_attempts
+    string_attr :failure_action
+    string_attr :exception_name
+    string_attr :exception_message
+    string_attr :stacktrace
+
+    timestamps
+
+    def enqueue
+      queue = AWS::SQS.new.queues[queue_name]
+      queue.send_message(id.to_s)
+    end
+
+    def enqueue_with_delay(delay)
+      queue = AWS::SQS.new.queues[queue_name]
+      queue.send_message(id.to_s, :delay_seconds => [900, [0, delay].max].min)
+    end
+
+    def work
+      begin
+        klass = constantize(model_class_name)
+        if model_id
+          target = klass.find(model_id)
+        else
+          target = klass
+        end
+        args = JSON.parse(job_arguments)
+        target.send(job_method, *args)
+        destroy
+      rescue StandardError => err
+        fail(err)
+      end
+    end
+
+    def fail(err)
+      self.failed_attempts ||= 0
+      self.failed_attempts += 1
+      self.exception_name = err.class.to_s
+      self.exception_message = err.message
+      self.stacktrace = err.backtrace[0, 7].join("\n")
+
+      if failure_action == 'retry' && retry_delay
+        enqueue_with_delay(retry_delay)
+      else
+        self.failed = true
+      end
+
+      self.save!
+    end
+
+    def retry_delay
+      return nil if failed_attempts.nil? || failed_attempts > 4
+      60 * 2**(failed_attempts - 1)
+    end
+
+    # from ActiveSupport source: http://api.rubyonrails.org/classes/ActiveSupport/Inflector.html#method-i-constantize
+    def constantize(camel_cased_word) # :nodoc:
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        if constant == Object
+          constant.const_get(name)
+        else
+          candidate = constant.const_get(name)
+          next candidate if constant.const_defined?(name, false)
+          next candidate unless Object.const_defined?(name)
+
+          # Go down the ancestors to check it it's owned
+          # directly before we reach Object or the end of ancestors.
+          constant = constant.ancestors.inject do |const, ancestor|
+            break const    if ancestor == Object
+            break ancestor if ancestor.const_defined?(name, false)
+            const
+          end
+
+          # owner is in Object, so raise
+          constant.const_get(name, false)
+        end
+      end
+    end
+
+  end
+
+end

data/lib/boss_queue.rb

@@ -0,0 +1,2 @@
+require 'boss_queue/boss_queue'
+require 'boss_queue/job'

data/spec/boss_queue_spec.rb

@@ -0,0 +1,244 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "BossQueue module" do
+
+  it "should respond to environment" do
+    BossQueue.should respond_to(:environment)
+  end
+
+  it "should respond to environment=" do
+    BossQueue.should respond_to(:environment=)
+  end
+
+  it "should respond to failure_action" do
+    BossQueue.should respond_to(:failure_action)
+  end
+
+  it "should respond to failure_action=" do
+    BossQueue.should respond_to(:failure_action=)
+  end
+
+  describe "#failure_action" do
+    it "should default to 'retry'" do
+      BossQueue.failure_action.should == 'retry'
+    end
+  end
+
+  describe "#table_name" do
+    before(:each) do
+      BossQueue.environment = nil
+    end
+
+    context "when @@environment is 'development'" do
+      it "should be 'dev_boss_queue_jobs'" do
+        BossQueue.environment = 'development'
+        BossQueue.table_name.should == 'dev_boss_queue_jobs'
+      end
+    end
+
+    context "when @@environment is 'production'" do
+      it "should be 'boss_queue_jobs'" do
+        BossQueue.environment = 'production'
+        BossQueue.table_name.should == 'boss_queue_jobs'
+      end
+    end
+
+    context "when @@environment is 'staging'" do
+      it "should be 'staging_boss_queue_jobs'" do
+        BossQueue.environment = 'staging'
+        BossQueue.table_name.should == 'staging_boss_queue_jobs'
+      end
+    end
+
+    context "when @@environment is 'staging'" do
+      it "should be 'staging_boss_queue_jobs'" do
+        BossQueue.environment = 'staging'
+        BossQueue.table_name.should == 'staging_boss_queue_jobs'
+      end
+    end
+
+    context "when @@environment is nil" do
+      it "should raise an exception" do
+        lambda {
+          BossQueue.table_name
+        }.should raise_error
+      end
+    end
+  end
+
+
+  describe "#queue_name" do
+    before(:each) do
+      BossQueue.environment = nil
+    end
+
+    context "when @@environment is 'development'" do
+      it "should be 'dev_boss_queue'" do
+        BossQueue.environment = 'development'
+        BossQueue.queue_name.should == 'dev_boss_queue'
+      end
+    end
+
+    context "when @@environment is 'production'" do
+      it "should be 'boss_queue'" do
+        BossQueue.environment = 'production'
+        BossQueue.queue_name.should == 'boss_queue'
+      end
+    end
+
+    context "when @@environment is 'staging'" do
+      it "should be 'staging_boss_queue'" do
+        BossQueue.environment = 'staging'
+        BossQueue.queue_name.should == 'staging_boss_queue'
+      end
+    end
+
+    context "when @@environment is 'staging'" do
+      it "should be 'staging_boss_queue'" do
+        BossQueue.environment = 'staging'
+        BossQueue.queue_name.should == 'staging_boss_queue'
+      end
+    end
+
+    context "when @@environment is nil" do
+      it "should raise an exception" do
+        lambda {
+          BossQueue.queue_name
+        }.should raise_error
+      end
+    end
+  end
+
+  describe "#enqueue" do
+    before(:each) do
+      @arguments = ['a', 'b', { 'c' => 2, 'd' => 1 }]
+      @argument_json = JSON.generate(@arguments)
+
+      class TestClass
+        def id
+          'xyz'
+        end
+
+        def self.test_class_method
+        end
+
+        def test_instance_method
+        end
+      end
+    end
+
+    context "when a class" do
+      it "should initialize a new BossQueue::Job object, save and call enqueue on it" do
+        BossQueue.environment = 'test'
+        BossQueue.failure_action = 'retry'
+        BossQueue::Job.any_instance.should_receive(:kind=).with('TestClass@test_class_method')
+        BossQueue::Job.any_instance.should_receive(:queue_name=).with('test_boss_queue')
+        BossQueue::Job.any_instance.should_receive(:failure_action=).with('retry')
+        BossQueue::Job.any_instance.should_receive(:model_class_name=).with('TestClass')
+        BossQueue::Job.any_instance.should_not_receive(:model_id=)
+        BossQueue::Job.any_instance.should_receive(:job_method=).with('test_class_method')
+        BossQueue::Job.any_instance.should_receive(:job_arguments=).with(@argument_json)
+        BossQueue::Job.any_instance.should_receive(:save!)
+        BossQueue::Job.any_instance.should_receive(:enqueue)
+        BossQueue.enqueue(TestClass, :test_class_method, 'a', 'b', { 'c' => 2, 'd' => 1 })
+      end
+    end
+
+    context "when a class instance" do
+      it "should initialize a new BossQueue::Job object, save and call enqueue on it" do
+        BossQueue.environment = 'test'
+        BossQueue.failure_action = 'retry'
+        BossQueue::Job.any_instance.should_receive(:kind=).with('TestClass#test_instance_method')
+        BossQueue::Job.any_instance.should_receive(:queue_name=).with('test_boss_queue')
+        BossQueue::Job.any_instance.should_receive(:failure_action=).with('retry')
+        BossQueue::Job.any_instance.should_receive(:model_class_name=).with('TestClass')
+        BossQueue::Job.any_instance.should_receive(:model_id=).with('xyz')
+        BossQueue::Job.any_instance.should_receive(:job_method=).with('test_instance_method')
+        BossQueue::Job.any_instance.should_receive(:job_arguments=).with(@argument_json)
+        BossQueue::Job.any_instance.should_receive(:save!)
+        BossQueue::Job.any_instance.should_receive(:enqueue)
+        BossQueue.enqueue(TestClass.new, :test_instance_method, 'a', 'b', { 'c' => 2, 'd' => 1 })
+      end
+    end
+
+  end
+
+  describe "#enqueue_with_delay" do
+    before(:each) do
+      @arguments = ['a', 'b', { 'c' => 2, 'd' => 1 }]
+      @argument_json = JSON.generate(@arguments)
+    end
+
+    context "when a class" do
+      it "should initialize a new BossQueue::Job object, save and call enqueue on it" do
+        BossQueue.environment = 'test'
+        BossQueue.failure_action = 'retry'
+        BossQueue::Job.any_instance.should_receive(:kind=).with('TestClass@test_class_method')
+        BossQueue::Job.any_instance.should_receive(:queue_name=).with('test_boss_queue')
+        BossQueue::Job.any_instance.should_receive(:failure_action=).with('retry')
+        BossQueue::Job.any_instance.should_receive(:model_class_name=).with('TestClass')
+        BossQueue::Job.any_instance.should_not_receive(:model_id=)
+        BossQueue::Job.any_instance.should_receive(:job_method=).with('test_class_method')
+        BossQueue::Job.any_instance.should_receive(:job_arguments=).with(@argument_json)
+        BossQueue::Job.any_instance.should_receive(:save!)
+        BossQueue::Job.any_instance.should_receive(:enqueue_with_delay).with(60)
+        BossQueue.enqueue_with_delay(60, TestClass, :test_class_method, 'a', 'b', { 'c' => 2, 'd' => 1 })
+      end
+    end
+
+    context "when a class instance" do
+      it "should initialize a new BossQueue::Job object, save and call enqueue on it" do
+        BossQueue.environment = 'test'
+        BossQueue.failure_action = 'retry'
+        BossQueue::Job.any_instance.should_receive(:kind=).with('TestClass#test_instance_method')
+        BossQueue::Job.any_instance.should_receive(:queue_name=).with('test_boss_queue')
+        BossQueue::Job.any_instance.should_receive(:failure_action=).with('retry')
+        BossQueue::Job.any_instance.should_receive(:model_class_name=).with('TestClass')
+        BossQueue::Job.any_instance.should_receive(:model_id=).with('xyz')
+        BossQueue::Job.any_instance.should_receive(:job_method=).with('test_instance_method')
+        BossQueue::Job.any_instance.should_receive(:job_arguments=).with(@argument_json)
+        BossQueue::Job.any_instance.should_receive(:save!)
+        BossQueue::Job.any_instance.should_receive(:enqueue_with_delay).with(60)
+        BossQueue.enqueue_with_delay(60, TestClass.new, :test_instance_method, 'a', 'b', { 'c' => 2, 'd' => 1 })
+      end
+    end
+
+  end
+
+  describe "#work" do
+    before(:each) do
+      @queue = double('queue')
+      AWS::SQS.stub_chain(:new, :queues, :[]).and_return(@queue)
+
+      @sqs_message = double('message')
+      @sqs_message.stub(:body).and_return('ijk')
+      @queue.stub(:receive_message).and_yield(@sqs_message)
+
+      @job = double('job')
+      @job.stub(:work)
+      @job.stub(:queue_name=)
+      BossQueue::Job.stub_chain(:shard, :find_by_id).and_return(@job)
+    end
+
+    it "should dequeue from SQS" do
+      @queue.should_receive(:receive_message).and_yield(@sqs_message)
+      BossQueue.work
+    end
+
+    context "when something is dequeued from SQS" do
+      it "should use the dequeued id to retrieve a BossQueue::Job object" do
+        @queue.should_receive(:receive_message).and_yield(@sqs_message)
+        shard = double('shard')
+        BossQueue::Job.should_receive(:shard).with(BossQueue.table_name).and_return(shard)
+        shard.should_receive(:find_by_id).with('ijk').and_return(@job)
+        BossQueue.work
+      end
+
+      it "should call work on the BossQueue::Job object" do
+        @job.should_receive(:work)
+        BossQueue.work
+      end
+    end
+  end
+
+end

data/spec/job_spec.rb

@@ -0,0 +1,358 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "BossQueue::Job" do
+
+  it "should respond to id" do
+    BossQueue::Job.new.should respond_to(:id)
+  end
+
+  it "should respond to id=" do
+    BossQueue::Job.new.should respond_to(:id=)
+  end
+
+
+  it "should respond to failed" do
+    BossQueue::Job.new.should respond_to(:failed)
+  end
+
+  it "should respond to failed=" do
+    BossQueue::Job.new.should respond_to(:failed=)
+  end
+
+  describe "#failed" do
+    it "should default to false" do
+      BossQueue::Job.new.failed.should be_false
+    end
+  end
+
+  it "should respond to queue_name" do
+    BossQueue::Job.new.should respond_to(:queue_name)
+  end
+
+  it "should respond to queue_name=" do
+    BossQueue::Job.new.should respond_to(:queue_name=)
+  end
+
+  it "should respond to failed_attempts" do
+    BossQueue::Job.new.should respond_to(:failed_attempts)
+  end
+
+  it "should respond to failed_attempts=" do
+    BossQueue::Job.new.should respond_to(:failed_attempts=)
+  end
+
+
+  it "should respond to failure_action" do
+    BossQueue::Job.new.should respond_to(:failure_action)
+  end
+
+  it "should respond to failure_action=" do
+    BossQueue::Job.new.should respond_to(:failure_action=)
+  end
+
+
+  it "should respond to exception_name" do
+    BossQueue::Job.new.should respond_to(:exception_name)
+  end
+
+  it "should respond to exception_name=" do
+    BossQueue::Job.new.should respond_to(:exception_name=)
+  end
+
+
+  it "should respond to exception_message" do
+    BossQueue::Job.new.should respond_to(:exception_message)
+  end
+
+  it "should respond to exception_message=" do
+    BossQueue::Job.new.should respond_to(:exception_message=)
+  end
+
+
+  it "should respond to stacktrace" do
+    BossQueue::Job.new.should respond_to(:stacktrace)
+  end
+
+  it "should respond to stacktrace=" do
+    BossQueue::Job.new.should respond_to(:stacktrace=)
+  end
+
+
+  it "should respond to model_class_name" do
+    BossQueue::Job.new.should respond_to(:model_class_name)
+  end
+
+  it "should respond to model_class_name=" do
+    BossQueue::Job.new.should respond_to(:model_class_name=)
+  end
+
+
+  it "should respond to model_id" do
+    BossQueue::Job.new.should respond_to(:model_id)
+  end
+
+  it "should respond to model_id=" do
+    BossQueue::Job.new.should respond_to(:model_id=)
+  end
+
+
+  it "should respond to job_method" do
+    BossQueue::Job.new.should respond_to(:job_method)
+  end
+
+  it "should respond to job_method=" do
+    BossQueue::Job.new.should respond_to(:job_method=)
+  end
+
+
+  it "should respond to job_arguments" do
+    BossQueue::Job.new.should respond_to(:job_arguments)
+  end
+
+  it "should respond to job_arguments=" do
+    BossQueue::Job.new.should respond_to(:job_arguments=)
+  end
+
+
+  describe "#work" do
+    before(:each) do
+      @job = BossQueue::Job.new
+      @job.stub(:destroy)
+      @job.model_class_name = 'TestClass'
+      @job.model_id = 'xyz'
+      @job.job_method = 'test_instance_method'
+      @arguments = ['a', 'b', { 'c' => 2, 'd' => 1 }]
+      @argument_json = JSON.generate(@arguments)
+      @job.job_arguments = @argument_json
+      @instance_to_work_on = double('instance_to_work_on')
+      @instance_to_work_on.stub(:test_instance_method)
+      TestClass.stub(:find).and_return(@instance_to_work_on)
+    end
+
+
+    context "when model_id is not nil" do
+      it "should use #find on the model class to instantiate an object to work on" do
+        TestClass.should_receive(:find).with('xyz').and_return(@instance_to_work_on)
+        @job.work
+      end
+
+      it "should pass the job arguments to the job method" do
+        @instance_to_work_on.should_receive(:test_instance_method).with('a', 'b', { 'c' => 2, 'd' => 1 })
+        @job.work
+      end
+    end
+
+    context "when model_id is nil" do
+      before(:each) do
+        @job = BossQueue::Job.new
+        @job.stub(:destroy)
+        @job.model_class_name = 'TestClass'
+        @job.job_method = 'test_class_method'
+        @arguments = ['a', 'b', { 'c' => 2, 'd' => 1 }]
+        @argument_json = JSON.generate(@arguments)
+        @job.job_arguments = @argument_json
+      end
+
+      it "should pass the job arguments to the job method on the class" do
+        TestClass.should_receive(:test_class_method).with('a', 'b', { 'c' => 2, 'd' => 1 })
+        @job.work
+      end
+    end
+
+    context "when the job method doesn't raise an exception" do
+      it "should call destroy" do
+        @job.should_receive(:destroy)
+        @job.work
+      end
+    end
+
+    context "when the job method raises an exception" do
+      before(:each) do
+        @instance_to_work_on.stub(:test_instance_method).and_raise(StandardError.new)
+        @job.stub(:fail)
+      end
+
+      it "should call fail" do
+        @job.should_receive(:fail)
+        @job.work
+      end
+
+      it "should not call destroy" do
+        @job.should_not_receive(:destroy)
+        @job.work
+      end
+
+      it "should not raise an exception" do
+        lambda {
+          @job.work
+        }.should_not raise_error
+      end
+    end
+
+  end
+
+  describe "#enqueue" do
+    it "should enqueue id into the SQS queue" do
+      queue = double('queue')
+      AWS::SQS.stub_chain(:new, :queues, :[]).and_return(queue)
+      queue.should_receive(:send_message).with('ijk')
+      job = BossQueue::Job.new
+      job.id = 'ijk'
+      job.enqueue
+    end
+  end
+
+  describe "#enqueue_with_delay" do
+    it "should enqueue id into the SQS queue with a delay" do
+      queue = double('queue')
+      AWS::SQS.stub_chain(:new, :queues, :[]).and_return(queue)
+      queue.should_receive(:send_message).with('ijk', :delay_seconds => 60)
+      job = BossQueue::Job.new
+      job.id = 'ijk'
+      job.enqueue_with_delay(60)
+    end
+
+    it "should limit the delay to 15 minutes" do
+      queue = double('queue')
+      AWS::SQS.stub_chain(:new, :queues, :[]).and_return(queue)
+      queue.should_receive(:send_message).with('ijk', :delay_seconds => 900)
+      job = BossQueue::Job.new
+      job.id = 'ijk'
+      job.enqueue_with_delay(10000)
+    end
+
+    it "should set a negative delay to 0" do
+      queue = double('queue')
+      AWS::SQS.stub_chain(:new, :queues, :[]).and_return(queue)
+      queue.should_receive(:send_message).with('ijk', :delay_seconds => 0)
+      job = BossQueue::Job.new
+      job.id = 'ijk'
+      job.enqueue_with_delay(-60)
+    end
+  end
+
+  describe "#retry_delay" do
+    before(:each) do
+      @job = BossQueue::Job.new
+    end
+
+    context "when failed_attempts is nil" do
+      it "should be nil" do
+        @job.retry_delay.should == nil
+      end
+    end
+
+    context "when failed_attempts is 1" do
+      it "should be 60" do
+        @job.failed_attempts = 1
+        @job.retry_delay.should == 60
+      end
+    end
+
+    context "when failed_attempts is 2" do
+      it "should be 120" do
+        @job.failed_attempts = 2
+        @job.retry_delay.should == 120
+      end
+    end
+
+    context "when failed_attempts is 3" do
+      it "should be 240" do
+        @job.failed_attempts = 3
+        @job.retry_delay.should == 240
+      end
+    end
+
+    context "when failed_attempts is 4" do
+      it "should be 480" do
+        @job.failed_attempts = 4
+        @job.retry_delay.should == 480
+      end
+    end
+
+    context "when failed_attempts greater than 4" do
+      it "should be nil ((60 + 120 + 240 + 480) == 900 (15 minutes), the maximum delay supported by SQS)" do
+        @job.failed_attempts = 5
+        @job.retry_delay.should be_nil
+      end
+    end
+  end
+
+  describe "#fail" do
+    before(:each) do
+      @job = BossQueue::Job.new
+      @job.stub(:retry_delay).and_return(nil)
+      @job.stub(:save!)
+      @job.stub(:enqueue_with_delay)
+      @err
+      begin
+        raise StandardError.new('hello world')
+      rescue StandardError => err
+        @err = err
+      end
+    end
+
+    context "when failed_attempts is a number" do
+      it "should increment failed_attempts" do
+        @job.failed_attempts = 1
+        @job.fail(@err)
+        @job.failed_attempts.should == 2
+      end
+    end
+
+    it "should store the exception, message, and the first 7 lines of the stacktrace in the BossQueue::Job object" do
+      @job.fail(@err)
+      @job.exception_name.should == @err.class.to_s
+      @job.exception_message.should == @err.message
+      @job.stacktrace.should == @err.backtrace[0, 7].join("\n")
+    end
+
+    it "should call save!" do
+      @job.should_receive(:save!)
+      @job.fail(@err)
+    end
+
+    context "when failure_action is 'retry'" do
+      before(:each) do
+        @job.failure_action = 'retry'
+      end
+
+      context "when retry_delay returns a number" do
+        it "should re-enqueue with that delay" do
+          @job.stub(:retry_delay).and_return(60)
+          @job.should_receive(:enqueue_with_delay).with(60)
+          @job.fail(@err)
+        end
+
+        context "when failed_attempts is nil" do
+          it "should set failed_attempts to 1" do
+            @job.fail(@err)
+            @job.failed_attempts.should == 1
+          end
+        end
+      end
+
+      context "when retry_delay returns nil" do
+        it "should not re-enqueue" do
+          @job.should_not_receive(:enqueue)
+          @job.should_not_receive(:enqueue_with_delay)
+          @job.fail(@err)
+        end
+
+        it "should set failed to true" do
+          @job.fail(@err)
+          @job.failed.should be_true
+        end
+      end
+    end
+
+    context "when failure_action is not 'retry'" do
+      it "should not re-enqueue" do
+        @job.should_not_receive(:enqueue)
+        @job.should_not_receive(:enqueue_with_delay)
+        @job.fail(@err)
+      end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,31 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'bundler'
+Bundler.require
+require 'pry'
+require 'pry-nav'
+require 'pry-stack_explorer'
+
+require 'boss_queue'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+
+
+  class TestClass
+    def id
+      'xyz'
+    end
+
+    def self.test_class_method
+    end
+
+    def test_instance_method
+    end
+  end
+
+end

metadata

@@ -0,0 +1,175 @@
+--- !ruby/object:Gem::Specification
+name: boss_queue
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Daniel Nelson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-stack_explorer
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A fault tolerant job queue built around Amazon SQS & DynamoDB
+email: daniel@populr.me
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.md
+files:
+- .rspec
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- lib/boss_queue.rb
+- lib/boss_queue/boss_queue.rb
+- lib/boss_queue/job.rb
+- spec/boss_queue_spec.rb
+- spec/job_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/populr/boss_queue
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1086179720775375384
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: A fault tolerant job queue built around Amazon SQS & DynamoDB
+test_files: []