chore-core 1.5.10 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1aae723bbfe580eb5d31876ca91f32dd28baae7
-  data.tar.gz: 014aae64a27e2a6eb6210b76561e6cc1765056b6
+  metadata.gz: 74242dc2e704cdf2cf5baf9dd9b25822bbe20066
+  data.tar.gz: 0082dc5ab748faff7ba73a8bb6f01773aaf6b35c
 SHA512:
-  metadata.gz: 110b80593989c4ddd8bf1502705d1ce88fd26ba0e458dcffdd9072ddcbf2d3dd9b7b4760b3a7ab37bbd150f66b4acea02e280e7396c2e203330820e5dceb6939
-  data.tar.gz: ef5522be2d88a900151437d3a127b5a692a7e71a3a77169a3fe12e25089157b363f7a0f890d1f48d4fd84aca305eaacfbb5f1553026a02a92ce99ef159ccf4c9
+  metadata.gz: de6ce3260f50d5a324843f6bbc9511974da8d69731e893aad11a5404ee4107f167a4cb02b9579e372f822f414551ce4ee754139a316845282a2465170f5c03a1
+  data.tar.gz: fdd9bb38cb35ff082e055bf76a897571bdde176cb1fa20cd9880ed19f57c55d908f0f3c6c830fceae62cec88f761c0228dcb1bcdb051f1e561a48ea5d3f2bc49

data/README.md

@@ -11,7 +11,7 @@ The full docs for Chore can always be found at http://tapjoy.github.io/chore.
 
 Chore can be integrated with any Ruby-based project by following these instructions:
 
-    gem 'chore-core', '~> 1.5'
+    gem 'chore-core', '~> 1.6.0'
 
 If you also plan on using SQS, you must also bring in dalli to use for memcached:
 
@@ -21,7 +21,7 @@ Create a `Chorefile` file in the root of your project directory. While you can c
 
     --require=./<FILE_TO_LOAD>
 
-Make sure that `--require` points to the main entry point for your app. If integrating with a Rails app, just point it to the directory of your application and it will handle loading the correct files on its own. 
+Make sure that `--require` points to the main entry point for your app. If integrating with a Rails app, just point it to the directory of your application and it will handle loading the correct files on its own.
 
 Other options include:
 
@@ -36,7 +36,7 @@ Other options include:
     --queue_prefix prefixy # A prefix to prepend to queue names, mainly for development and qa testing purposes
     --max-attempts 100 # The maximum number of times a job can be attempted
     --dupe-on-cache-failure # Determines the deduping behavior when a cache connection error occurs. When set to `false`, the message is assumed not to be a duplicate. Defaults to `false`.
-    --queue-polling-size 10 # If your particular queueing system supports responding with messages in batches of a certain size, you can control that with this flag. SQS has a built in upper-limit of 10, but other systems will vary. 
+    --queue-polling-size 10 # If your particular queueing system supports responding with messages in batches of a certain size, you can control that with this flag. SQS has a built in upper-limit of 10, but other systems will vary.
 
 If you're using SQS, you'll want to add AWS keys so that Chore can authenticate with AWS.
 
@@ -62,7 +62,7 @@ Because you are likely to use the same app as the basis for both producing and c
 
 However, like many aspects of Chore, it is ultimately up to the developer to decide which use case fits their needs best. Chore is happy to let you configure it in almost any way you want.
 
-An example of how to configure chore via and initializer:
+An example of how to configure chore via an initializer:
 
 ```ruby
 Chore.configure do |c|
@@ -139,6 +139,28 @@ It is worth noting that any option that can be set via config file or command-li
 
 If a global publisher is set, it can be overridden on a per-job basis by specifying the publisher in `queue_options`.
 
+## Retry Backoff Strategy
+
+Chore has basic support for delaying retries  of a failed job using a step function. Currently the only queue that
+supports this functionality is SQS, all others will simply ignore the delay setting.
+
+### Setup
+
+The `:backoff` option for a queue expects a lambda that takes a single `UnitOfWork` argument. The return should be a
+number of seconds to delay the next attempt.
+
+```ruby
+queue_options :name => 'nameOfQueue',
+  :backoff => lambda { |work| work.current_attempt ** 2 } # Exponential backoff
+```
+
+### Using the Backoff
+
+If there is a `:backoff` option supplied, any failures will delay the next attempt by the result of that lambda.
+
+### Notes on SQS and Delays
+
+Read more details about SQS and Delays [here](docs/Delayed Jobs.md)
 
 ## Hooks
 
@@ -182,7 +204,11 @@ class TestJob
   queue_options :name => 'test_queue'
 
   def perform(args={})
-    Chore.logger.debug "My first sync job"
+    # Do something cool
+  end
+  
+  def before_perform_log(message)
+    Chore.logger.debug "About to do something cool with: #{message.inspect}"
   end
 end
 ```
@@ -257,5 +283,5 @@ Chore has several plugin gems available, which extend it's core functionality
 
 ## Copyright
 
-Copyright (c) 2013 - 2014 Tapjoy. See LICENSE.txt for
+Copyright (c) 2013 - 2015 Tapjoy. See LICENSE.txt for
 further details.

data/bin/chore

@@ -22,6 +22,9 @@ Chore::Signal.trap "USR1" do
 end
 
 begin
+  # Pre-load any Bundler dependencies now, so that the CLI parser has them loaded
+  # prior to intrpretting the command line args for things like consumers/producers
+  Bundler.require if defined?(Bundler)
   cli = Chore::CLI.instance
   cli.run!(ARGV)
 rescue => e
@@ -30,5 +33,3 @@ rescue => e
   STDERR.puts e.backtrace.join("\n")
   exit 1
 end
- 
-

data/chore-core.gemspec

@@ -37,9 +37,9 @@ Gem::Specification.new do |s|
   s.summary = "Job processing... for the future!"
 
   s.add_runtime_dependency(%q<json>, [">= 0"])
-  s.add_runtime_dependency(%q<aws-sdk>, ["~> 1.56", ">= 1.56.0"])
+  s.add_runtime_dependency(%q<aws-sdk-v1>, ["~> 1.56", ">= 1.56.0"])
   s.add_runtime_dependency(%q<thread>, ["~> 0.1.3"])
-  s.add_development_dependency(%q<rspec>, ["~> 2.12.0"])
+  s.add_development_dependency(%q<rspec>, ["~> 3.3.0"])
   s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
   s.add_development_dependency(%q<bundler>, [">= 0"])
 end

data/lib/chore/job.rb

@@ -16,7 +16,7 @@ module Chore
       # Throw a RejectMessageException from your job to signal that the message should be rejected.
       # The semantics of +reject+ are queue implementation dependent.
     end
-    
+
     def self.job_classes #:nodoc:
       @classes || []
     end
@@ -42,15 +42,24 @@ module Chore
 
     module ClassMethods
       DEFAULT_OPTIONS = { }
-      
+
       # Pass a hash of options to queue_options the included class's use of Chore::Job
-      # +opts+ has just the one required option. 
+      # +opts+ has just the one required option.
       # * +:name+: which should map to the name of the queue this job should be published to.
       def queue_options(opts = {})
         @chore_options = (@chore_options || DEFAULT_OPTIONS).merge(opts_from_cli).merge(opts)
+
         required_options.each do |k|
           raise ArgumentError.new("#{self.to_s} :#{k} is a required option for Chore::Job") unless @chore_options[k]
         end
+
+        if @chore_options.key?(:backoff)
+          if !@chore_options[:backoff].is_a?(Proc)
+            raise ArgumentError, "#{self.to_s}: backoff must be a lambda or Proc"
+          elsif @chore_options[:backoff].arity != 1
+            raise ArgumentError, "#{self.to_s}: backoff must accept a single argument"
+          end
+        end
       end
 
       # This is a method so it can be overriden to create additional required
@@ -68,14 +77,14 @@ module Chore
         @from_cli ||= (Chore.config.marshal_dump.select {|k,v| required_options.include? k } || {})
       end
 
-      # Execute the current job. We create an instance of the job to do the perform 
+      # Execute the current job. We create an instance of the job to do the perform
       # as this allows the jobs themselves to do initialization that might require access
       # to the parameters of the job.
       def perform(*args)
         job = self.new(args)
         job.perform(*args)
       end
-      
+
       # Publish a job using an instance of job. Similar to perform we do this so that a job
       # can perform initialization logic before the perform_async is begun. This, in addition, to
       # hooks allows for rather complex jobs to be written simply.
@@ -93,6 +102,12 @@ module Chore
       def prefixed_queue_name
         "#{Chore.config.queue_prefix}#{self.options[:name]}"
       end
+
+      # We require a proc for the backoff strategy, but as we check for it in `.queue_options` we can simply check for
+      # the key at this point.
+      def has_backoff?
+        self.options.key?(:backoff)
+      end
     end #ClassMethods
 
     # This is handy to override in an included job to be able to do job setup that requires

data/lib/chore/queues/sqs.rb

@@ -3,10 +3,23 @@ module Chore
     module SQS
       # Helper method to create queues based on the currently known list as provided by your configured Chore::Jobs
       # This is meant to be invoked from a rake task, and not directly.
-      # These queues will be created with the default settings, which may not be ideal. 
+      # These queues will be created with the default settings, which may not be ideal.
       # This is meant only as a convenience helper for testing, and not as a way to create production quality queues in SQS
-      def self.create_queues!
+      def self.create_queues!(halt_on_existing=false)
         raise 'You must have atleast one Chore Job configured and loaded before attempting to create queues' unless Chore.prefixed_queue_names.length > 0
+
+        if halt_on_existing
+          existing = self.existing_queues
+          if existing.size > 0
+            raise <<-ERROR.gsub(/^\s+/, '')
+            We found queues that already exist! Verify your queue names or prefix are setup correctly.
+
+            The following queue names were found:
+            #{existing.join("\n")}
+            ERROR
+          end
+        end
+
         #This will raise an exception if AWS has not been configured by the project making use of Chore
         sqs_queues = AWS::SQS.new.queues
         Chore.prefixed_queue_names.each do |queue_name|
@@ -38,6 +51,22 @@ module Chore
         end
         Chore.prefixed_queue_names
       end
+
+      # Collect a list of queues that already exist
+      def self.existing_queues
+        #This will raise an exception if AWS has not been configured by the project making use of Chore
+        sqs_queues = AWS::SQS.new.queues
+
+        Chore.prefixed_queue_names.select do |queue_name|
+          # If the NonExistentQueue exception is raised we do not care about that queue name.
+          begin
+            sqs_queues.named(queue_name)
+            true
+          rescue AWS::SQS::Errors::NonExistentQueue
+            false
+          end
+        end
+      end
     end
   end
 end

data/lib/chore/queues/sqs/consumer.rb

@@ -55,6 +55,14 @@ module Chore
           queue.batch_delete([id])
         end
 
+        def delay(item, backoff_calc)
+          delay = backoff_calc.call(item)
+          Chore.logger.debug "Delaying #{item.id} by #{delay} seconds"
+          queue.batch_change_visibility(delay, [item.id])
+
+          return delay
+        end
+
         private
 
         # Requests messages from SQS, and invokes the provided +&block+ over each one. Afterwards, the :on_fetch
@@ -63,15 +71,17 @@ module Chore
           msg = queue.receive_messages(:limit => sqs_polling_amount, :attributes => [:receive_count])
           messages = *msg
           messages.each do |message|
-            block.call(message.handle, queue_name, queue_timeout, message.body, message.receive_count - 1) unless duplicate_message?({:id=>message.id, :queue=>message.queue.url, :visibility_timeout=>message.queue.visibility_timeout})
+            unless duplicate_message?(message)
+              block.call(message.handle, queue_name, queue_timeout, message.body, message.receive_count - 1)
+            end
             Chore.run_hooks_for(:on_fetch, message.handle, message.body)
           end
           messages
         end
 
         # Checks if the given message has already been received within the timeout window for this queue
-        def duplicate_message?(msg_data)
-          dupe_detector.found_duplicate?(msg_data)
+        def duplicate_message?(message)
+          dupe_detector.found_duplicate?(:id=>message.id, :queue=>message.queue.url, :visibility_timeout=>message.queue.visibility_timeout)
         end
 
         # Returns the instance of the DuplicateDetector used to ensure unique messages.

data/lib/chore/tasks/queues.task

@@ -1,7 +1,14 @@
 namespace :chore do
-  desc "Create all defined queues"
-  task :create do
-    Chore::Queues::SQS.create_queues!
+  desc <<-DESC.gsub(/^\s+/, '')
+  Create all defined queues. If the halt_on_existing argument is set (defaults to off) the task will abort if a single
+  queue already exists without attempting to create any.
+
+  This flag is specifically provided for our integration testing platform to ensure we don't deploy to an incorrect environment.
+  DESC
+  task :create, :halt_on_existing do |t, args|
+    halt_on_existing = %w(1 true yes t y).include?(args[:halt_on_existing])
+
+    Chore::Queues::SQS.create_queues!(halt_on_existing)
   end
 
   desc "Remove all defined queues"

data/lib/chore/version.rb

@@ -1,8 +1,8 @@
 module Chore
   module Version #:nodoc:
     MAJOR = 1
-    MINOR = 5
-    PATCH = 10
+    MINOR = 7
+    PATCH = 2
 
     STRING = [ MAJOR, MINOR, PATCH ].join('.')
   end

data/lib/chore/worker.rb

@@ -22,7 +22,7 @@ module Chore
 
     # Create a Worker. Give it an array of work (or single item), and +opts+.
     # Currently, the only option supported by Worker is +:payload_handler+ which contains helpers
-    # for decoding the item and finding the correct payload class 
+    # for decoding the item and finding the correct payload class
     def initialize(work=[],opts={})
       @stopping = false
       @started_at = Time.now
@@ -79,27 +79,44 @@ module Chore
       message = options[:payload_handler].decode(item.message)
       klass = options[:payload_handler].payload_class(message)
       return unless klass.run_hooks_for(:before_perform,message)
+
       begin
         Chore.logger.info { "Running job #{klass} with params #{message}"}
         perform_job(klass,message)
         item.consumer.complete(item.id)
         Chore.logger.info { "Finished job #{klass} with params #{message}"}
-        klass.run_hooks_for(:after_perform,message)
+        klass.run_hooks_for(:after_perform, message)
       rescue Job::RejectMessageException
         item.consumer.reject(item.id)
         Chore.logger.error { "Failed to run job for #{item.message}  with error: Job raised a RejectMessageException" }
         klass.run_hooks_for(:on_rejected, message)
       rescue => e
-        Chore.logger.error { "Failed to run job #{item.message} with error: #{e.message} at #{e.backtrace * "\n"}" }
-        if item.current_attempt >= klass.options[:max_attempts]
-          klass.run_hooks_for(:on_permanent_failure,item.queue_name,message,e)
-          item.consumer.complete(item.id)
+        if klass.has_backoff?
+          attempt_to_delay(item, message, klass)
         else
-          klass.run_hooks_for(:on_failure,message,e)
+          handle_failure(item, message, klass, e)
         end
       end
     end
 
+    def attempt_to_delay(item, message, klass)
+      delayed_for = item.consumer.delay(item, klass.options[:backoff])
+      Chore.logger.info { "Delaying retry by #{delayed_for} seconds for the job #{item.message}" }
+      klass.run_hooks_for(:on_delay, message)
+    rescue => e
+      handle_failure(item, message, klass, e)
+    end
+
+    def handle_failure(item, message, klass, e)
+      Chore.logger.error { "Failed to run job #{item.message} with error: #{e.message} at #{e.backtrace * "\n"}" }
+      if item.current_attempt >= klass.options[:max_attempts]
+        klass.run_hooks_for(:on_permanent_failure,item.queue_name,message,e)
+        item.consumer.complete(item.id)
+      else
+        klass.run_hooks_for(:on_failure, message, e)
+      end
+    end
+
     def perform_job(klass, message)
       klass.perform(*options[:payload_handler].payload(message))
     end

data/spec/chore/duplicate_detector_spec.rb

@@ -15,32 +15,32 @@ describe Chore::DuplicateDetector do
 
   describe "#found_duplicate" do
     it 'should not return true if the message has not already been seen' do
-      memcache.should_receive(:add).and_return(true)
-      dedupe.found_duplicate?(message_data).should_not be_true
+      expect(memcache).to receive(:add).and_return(true)
+      expect(dedupe.found_duplicate?(message_data)).to_not be true
     end
 
     it 'should return true if the message has already been seen' do
-      memcache.should_receive(:add).and_return(false)
-      dedupe.found_duplicate?(message_data).should be_true
+      expect(memcache).to receive(:add).and_return(false)
+      expect(dedupe.found_duplicate?(message_data)).to be true
     end
 
     it 'should return false if given an invalid message' do
-      dedupe.found_duplicate?({}).should be_false
+      expect(dedupe.found_duplicate?({})).to be false
     end
 
     it "should return false when identity store errors" do
-      memcache.should_receive(:add).and_raise("no")
-      dedupe.found_duplicate?(message_data).should be_false
+      expect(memcache).to receive(:add).and_raise("no")
+      expect(dedupe.found_duplicate?(message_data)).to be false
     end
 
     it "should set the timeout to be the queue's " do
-      memcache.should_receive(:add).with(id,"1",timeout).and_return(true)
-      dedupe.found_duplicate?(message_data).should be_false
+      expect(memcache).to receive(:add).with(id,"1",timeout).and_return(true)
+      expect(dedupe.found_duplicate?(message_data)).to be false
     end
 
     it "should call #visibility_timeout once and only once" do
-      queue.should_receive(:visibility_timeout).once
-      memcache.should_receive(:add).at_least(3).times.and_return(true)
+      expect(queue).to receive(:visibility_timeout).once
+      expect(memcache).to receive(:add).at_least(3).times.and_return(true)
       3.times { dedupe.found_duplicate?(message_data) }
     end
 
@@ -49,8 +49,8 @@ describe Chore::DuplicateDetector do
         let(:dupe_on_cache_failure) { true }
 
         it "returns true" do
-          memcache.should_receive(:add).and_raise
-          dedupe.found_duplicate?(message_data).should be_true
+          expect(memcache).to receive(:add).and_raise
+          expect(dedupe.found_duplicate?(message_data)).to be true
         end
       end
     end

data/spec/chore/job_spec.rb

@@ -9,7 +9,8 @@ describe Chore::Job do
   end
 
   after(:each) do
-    TestJob.queue_options config
+    # Reset the config
+    TestJob.instance_variable_set(:@chore_options, nil)
   end
 
   it 'should have an perform_async method' do
@@ -38,7 +39,30 @@ describe Chore::Job do
     TestJob.options[:name].should == 'test_queue'
   end
 
-  describe(:perform_async) do 
+  describe 'the backoff config' do
+    it 'must be a Proc instance' do
+      options = config.merge(:backoff => 'abc')
+
+      expect { TestJob.queue_options(options) }.to raise_error(ArgumentError)
+    end
+
+    it 'rejects a lambda with no arguments' do
+      options = config.merge(:backoff => lambda { })
+      expect { TestJob.queue_options(options) }.to raise_error(ArgumentError)
+    end
+
+    it 'allows a lambda with one argument' do
+      options = config.merge(:backoff => lambda { |a| })
+      expect { TestJob.queue_options(options) }.not_to raise_error
+    end
+
+    it 'rejects a lambda with two arguments' do
+      options = config.merge(:backoff => lambda { |a,b| })
+      expect { TestJob.queue_options(options) }.to raise_error(ArgumentError)
+    end
+  end
+
+  describe(:perform_async) do
     it 'should call an instance of the queue_options publisher' do
       args = [1,2,{:h => 'ash'}]
       TestJob.queue_options(:publisher => Chore::Publisher)