chore-core 3.2.3 → 4.0.0

data/chore-core.gemspec

@@ -37,11 +37,10 @@ Gem::Specification.new do |s|
   s.summary = "Job processing... for the future!"
 
   s.add_runtime_dependency(%q<json>, [">= 0"])
-  s.add_runtime_dependency(%q<aws-sdk-v1>, ["~> 1.56", ">= 1.56.0"])
+  s.add_runtime_dependency(%q<aws-sdk-sqs>, ["~> 1"])
   s.add_runtime_dependency(%q<thread>, ["~> 0.1.3"])
   s.add_runtime_dependency('get_process_mem', ["~> 0.2.0"])
-  s.add_development_dependency(%q<rspec>, ["~> 3.3.0"])
+  s.add_development_dependency(%q<rspec>, ["~> 3.3"])
   s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
   s.add_development_dependency(%q<bundler>, [">= 0"])
 end
-

data/lib/chore.rb

@@ -63,6 +63,24 @@ module Chore #:nodoc:
     end
   end
 
+  def self.log_level_to_sym
+    return self.config[:log_level] if self.config[:log_level].is_a?(Symbol)
+    case self.config[:log_level]
+    when 0
+      :debug
+    when 1
+      :info
+    when 2
+      :warn
+    when 3
+      :error
+    when 4
+      :fatal
+    else
+      :unknown
+    end
+  end
+
   # Reopens any open files.  This will match any logfile that was opened by Chore,
   # Rails, or any other library.
   def self.reopen_logs
@@ -218,6 +236,8 @@ module Chore #:nodoc:
   end
 
   # List of queue_names as configured via Chore::Job including their prefix, if set.
+  #
+  # @return [Array<String>]
   def self.prefixed_queue_names
     Chore::Job.job_classes.collect {|klass| c = constantize(klass); c.prefixed_queue_name}
   end

data/lib/chore/cli.rb

@@ -92,8 +92,8 @@ module Chore #:nodoc:
       validate_strategy!
     end
 
-
     private
+
     def setup_options #:nodoc:
       register_option "queues", "-q", "--queues QUEUE1,QUEUE2", "Names of queues to process (default: all known)" do |arg|
         # This will remove duplicates. We ultimately force this to be a Set further below
@@ -289,4 +289,3 @@ module Chore #:nodoc:
     end
   end
 end
-

data/lib/chore/configuration.rb

@@ -1,6 +1,6 @@
 module Chore
   # Wrapper around an OpenStruct to define configuration data
-  # (TODO): Add required opts, and validate that they're set
+  # TODO: Add required opts, and validate that they're set
   class Configuration < OpenStruct
     # Helper method to make merging Hashes into OpenStructs easier
     def merge_hash(hsh={})

data/lib/chore/consumer.rb

@@ -1,16 +1,17 @@
 module Chore
   # Raised when Chore is booting up, but encounters a set of configuration that is impossible to boot from. Typically
-  # you'll find additional information around the cause of the exception by examining the logfiles
+  # you'll find additional information around the cause of the exception by examining the logfiles.
+  # You can raise this exception if your queue is in a terrible state and must shut down.
   class TerribleMistake < Exception
-    # You can raise this exception if your queue is in a terrible state and must shut down
   end
 
-  # Base class for a Chore Consumer. Provides the basic interface to adhere to for building custom
-  # Chore Consumers.
+  # Base class for a Chore Consumer. Provides the interface that a Chore::Consumer implementation should adhere to.
   class Consumer
 
     attr_accessor :queue_name
 
+    # @param [String] queue_name Name of queue to be consumed from
+    # @param [Hash] opts
     def initialize(queue_name, opts={})
       @queue_name = queue_name
       @running = true
@@ -24,18 +25,25 @@ module Chore
     # Consume takes a block with an arity of two. The two params are
     # |message_id,message_body| where message_id is any object that the
     # consumer will need to be able to act on a message later (reject, complete, etc)
-    def consume(&block)
+    #
+    # @param [Block] &handler Message handler, used by the calling context (worker) to create & assigns a UnitOfWork
+    def consume(&handler)
       raise NotImplementedError
     end
 
     # Reject should put a message back on a queue to be processed again later. It takes
     # a message_id as returned via consume.
+    #
+    # @param [String] message_id Unique ID of the message
     def reject(message_id)
       raise NotImplementedError
     end
 
-    # Complete should mark a message as finished. It takes a message_id as returned via consume
-    def complete(message_id)
+    # Complete should mark a message as finished.
+    #
+    # @param [String] message_id Unique ID of the message
+    # @param [Hash] receipt_handle Unique ID of the consuming transaction in non-filesystem implementations
+    def complete(message_id, receipt_handle)
       raise NotImplementedError
     end
 
@@ -45,23 +53,47 @@ module Chore
     end
 
     # Returns true if the Consumer is currently running
+    #
+    # @return [TrueClass, FalseClass]
     def running?
       @running
     end
 
-    # returns up to n work
+    # Returns up to n work
+    #
+    # @param n
     def provide_work(n)
       raise NotImplementedError
     end
 
-    # now, given an arbitrary key and klass, have we seen the key already?
+    # Determine whether or not we have already seen this message
+    #
+    # @param [String] dedupe_key
+    # @param [Class] klass
+    # @param [Integer] queue_timeout
+    #
+    # @return [TrueClass, FalseClass]
     def duplicate_message?(dedupe_key, klass, queue_timeout)
       dupe_detector.found_duplicate?(:id=>dedupe_key, :queue=>klass.to_s, :visibility_timeout=>queue_timeout)
     end
 
+    # Instance of duplicate detection implementation class
+    #
+    # @return [DuplicateDetector]
     def dupe_detector
       @dupes ||= DuplicateDetector.new({:servers => Chore.config.dedupe_servers,
                                         :dupe_on_cache_failure => false})
     end
+
+    private
+
+    # Gets messages from queue implementation and invokes the provided block over each one. Afterwards, the :on_fetch
+    # hook will be invoked per message. This block call provides data necessary for the worker (calling context) to
+    # populate a UnitOfWork struct.
+    #
+    # @param [Block] &handler Message handler, passed along by #consume
+    def handle_messages(&handler)
+      raise NotImplementedError
+    end
   end
 end

data/lib/chore/job.rb

@@ -105,6 +105,8 @@ module Chore
       end
 
       # The name of the configured queue, combined with an optional prefix
+      #
+      # @return [String]
       def prefixed_queue_name
         "#{Chore.config.queue_prefix}#{self.options[:name]}"
       end

data/lib/chore/publisher.rb

@@ -1,26 +1,42 @@
 module Chore
-  # Base class for Chore Publishers. Provides the bare interface one needs to adhere to when writing custom publishers
+  # Base class for a Chore Publisher. Provides the interface that a Chore::Publisher implementation should adhere to.
   class Publisher
     DEFAULT_OPTIONS = { :encoder => Encoder::JsonEncoder }
 
     attr_accessor :options
 
+    # @param [Hash] opts
     def initialize(opts={})
       self.options = DEFAULT_OPTIONS.merge(opts)
     end
 
     # Publishes the provided +job+ to the queue identified by the +queue_name+. Not designed to be used directly, this
     # method ferries to the publish method on an instance of your configured Publisher.
+    #
+    # @param [String] queue_name Name of queue to be consumed from
+    # @param [Hash] job Job instance definition, will be encoded to JSON
     def self.publish(queue_name,job)
       self.new.publish(queue_name,job)
     end
 
-    # Raises a NotImplementedError. This method should be overridden in your descendent, custom publisher class
+    # Publishes a message to queue
+    #
+    # @param [String] queue_name Name of the SQS queue
+    # @param [Hash] job Job instance definition, will be encoded to JSON
     def publish(queue_name,job)
       raise NotImplementedError
     end
+
+    # Sets a flag that instructs the publisher to reset the connection the next time it's used.
+    # Should be overriden in publishers (but is not required)
+    def self.reset_connection!
+    end
+
   protected
 
+    # Encodes the job class to format provided by endoder implementation
+    #
+    # @param [Any] job
     def encode_job(job)
       options[:encoder].encode(job)
     end

data/lib/chore/queues/filesystem/consumer.rb

@@ -15,12 +15,12 @@ module Chore
       # Once complete job files are deleted.
       # If rejected they are moved back into new and will be processed again.  This may not be the
       # desired behavior long term and we may want to add configuration to this class to allow more
-      # creating failure handling and retrying. 
+      # creating failure handling and retrying.
       class Consumer < Chore::Consumer
         extend FilesystemQueue
 
         Chore::CLI.register_option 'fs_queue_root', '--fs-queue-root DIRECTORY', 'Root directory for fs based queue'
-        
+
         class << self
           # Cleans up expired in-progress files by making them new again.
           def cleanup(expiration_time, new_dir, in_progress_dir)
@@ -51,7 +51,7 @@ module Chore
 
             # If the file is non-zero, this means it was successfully written to
             # by a publisher and we can attempt to move it to "in progress".
-            # 
+            #
             # There is a small window of time where the file can be zero, but
             # the publisher hasn't finished writing to the file yet.
             if !File.zero?(from)
@@ -68,7 +68,7 @@ module Chore
               # The file is empty (zero bytes) and enough time has passed since
               # the file was written that we can safely assume it will never
               # get written to be the publisher.
-              # 
+              #
               # The scenario where this happens is when the publisher created
               # the file, but the process was killed before it had a chance to
               # actually write the data.
@@ -114,7 +114,7 @@ module Chore
 
         # The minimum number of seconds to allow to pass between checks for expired
         # jobs on the filesystem.
-        # 
+        #
         # Since queue times are measured on the order of seconds, 1 second is the
         # smallest duration.  It also prevents us from burning a lot of CPU looking
         # at expired jobs when the consumer sleep interval is less than 1 second.
@@ -144,7 +144,7 @@ module Chore
               end
 
               found_files = false
-              handle_jobs do |*args|
+              handle_messages do |*args|
                 found_files = true
                 yield(*args)
               end
@@ -161,9 +161,14 @@ module Chore
 
         end
 
-        def complete(id)
-          Chore.logger.debug "Completing (deleting): #{id}"
-          File.delete(File.join(@in_progress_dir, id))
+        # Deletes the given message from filesystem queue. Since the filesystem is not a remote API, there is no
+        # notion of a "receipt handle".
+        #
+        # @param [String] message_id Unique ID of the message
+        # @param [Hash] receipt_handle Receipt handle of the message. Always nil for the filesystem consumer
+        def complete(message_id, receipt_handle = nil)
+          Chore.logger.debug "Completing (deleting): #{message_id}"
+          File.delete(File.join(@in_progress_dir, message_id))
         rescue Errno::ENOENT
           # The job took too long to complete, was deemed expired, and moved
           # back into "new".  Ignore.
@@ -173,7 +178,7 @@ module Chore
 
         # finds all new job files, moves them to in progress and starts the job
         # Returns a list of the job files processed
-        def handle_jobs(&block)
+        def handle_messages(&block)
           self.class.each_file(@new_dir, Chore.config.queue_polling_size) do |job_file|
             Chore.logger.debug "Found a new job #{job_file}"
 
@@ -186,8 +191,9 @@ module Chore
             job_json = File.read(in_progress_path)
             basename, previous_attempts, * = self.class.file_info(job_file)
 
-            # job_file is just the name which is the job id
-            block.call(job_file, queue_name, queue_timeout, job_json, previous_attempts)
+            # job_file is just the name which is the job id. 2nd argument (:receipt_handle) is nil because the
+            # filesystem is dealt with directly, as opposed to being an external API
+            block.call(job_file, nil, queue_name, queue_timeout, job_json, previous_attempts)
             Chore.run_hooks_for(:on_fetch, job_file, job_json)
           end
         end
@@ -203,4 +209,3 @@ module Chore
     end
   end
 end
-

data/lib/chore/queues/filesystem/publisher.rb

@@ -11,7 +11,7 @@ module Chore
         include FilesystemQueue
 
         # use of mutex and file locking should make this both threadsafe and safe for multiple
-        # processes to use the same queue directory simultaneously. 
+        # processes to use the same queue directory simultaneously.
         def publish(queue_name,job)
           # First try encoding the job to avoid writing empty job files if this fails
           encoded_job = encode_job(job)

data/lib/chore/queues/sqs.rb

@@ -1,10 +1,18 @@
 module Chore
   module Queues
     module SQS
+      def self.sqs_client
+        Aws::SQS::Client.new(logger: Chore.logger, log_level: Chore.log_level_to_sym)
+      end
+
       # Helper method to create queues based on the currently known list as provided by your configured Chore::Jobs
       # This is meant to be invoked from a rake task, and not directly.
       # These queues will be created with the default settings, which may not be ideal.
       # This is meant only as a convenience helper for testing, and not as a way to create production quality queues in SQS
+      #
+      # @param [TrueClass, FalseClass] halt_on_existing Raise an exception if the queue already exists
+      #
+      # @return [Array<String>]
       def self.create_queues!(halt_on_existing=false)
         raise 'You must have atleast one Chore Job configured and loaded before attempting to create queues' unless Chore.prefixed_queue_names.length > 0
 
@@ -20,49 +28,50 @@ module Chore
           end
         end
 
-        #This will raise an exception if AWS has not been configured by the project making use of Chore
-        sqs_queues = AWS::SQS.new.queues
         Chore.prefixed_queue_names.each do |queue_name|
           Chore.logger.info "Chore Creating Queue: #{queue_name}"
           begin
-            sqs_queues.create(queue_name)
-          rescue AWS::SQS::Errors::QueueAlreadyExists
+            sqs_client.create_queue(queue_name: queue_name)
+          rescue Aws::SQS::Errors::QueueAlreadyExists
             Chore.logger.info "exists with different config"
           end
         end
+
         Chore.prefixed_queue_names
       end
 
       # Helper method to delete all known queues based on the list as provided by your configured Chore::Jobs
       # This is meant to be invoked from a rake task, and not directly.
+      #
+      # @return [Array<String>]
+
       def self.delete_queues!
         raise 'You must have atleast one Chore Job configured and loaded before attempting to create queues' unless Chore.prefixed_queue_names.length > 0
-        #This will raise an exception if AWS has not been configured by the project making use of Chore
-        sqs_queues = AWS::SQS.new.queues
+
         Chore.prefixed_queue_names.each do |queue_name|
           begin
             Chore.logger.info "Chore Deleting Queue: #{queue_name}"
-            url = sqs_queues.url_for(queue_name)
-            sqs_queues[url].delete
+            url = sqs_client.get_queue_url(queue_name: queue_name).queue_url
+            sqs_client.delete_queue(queue_url: url)
           rescue => e
             # This could fail for a few reasons - log out why it failed, then continue on
             Chore.logger.error "Deleting Queue: #{queue_name} failed because #{e}"
           end
         end
+
         Chore.prefixed_queue_names
       end
 
       # Collect a list of queues that already exist
+      #
+      # @return [Array<String>]
       def self.existing_queues
-        #This will raise an exception if AWS has not been configured by the project making use of Chore
-        sqs_queues = AWS::SQS.new.queues
-
         Chore.prefixed_queue_names.select do |queue_name|
           # If the NonExistentQueue exception is raised we do not care about that queue name.
           begin
-            sqs_queues.named(queue_name)
+            sqs_client.get_queue_url(queue_name: queue_name)
             true
-          rescue AWS::SQS::Errors::NonExistentQueue
+          rescue Aws::SQS::Errors::NonExistentQueue
             false
           end
         end

data/lib/chore/queues/sqs/consumer.rb

@@ -1,9 +1,6 @@
-require 'aws/sqs'
+require 'aws-sdk-sqs'
 require 'chore/duplicate_detector'
 
-AWS.eager_autoload! AWS::Core
-AWS.eager_autoload! AWS::SQS
-
 module Chore
   module Queues
     module SQS
@@ -17,24 +14,29 @@ module Chore
         Chore::CLI.register_option 'aws_secret_key', '--aws-secret-key KEY', 'Valid AWS Secret Key'
         Chore::CLI.register_option 'dedupe_servers', '--dedupe-servers SERVERS', 'List of mememcache compatible server(s) to use for storing SQS Message Dedupe cache'
 
+        # @param [String] queue_name Name of SQS queue
+        # @param [Hash] opts Options
         def initialize(queue_name, opts={})
           super(queue_name, opts)
-
           raise Chore::TerribleMistake, "Cannot specify a queue polling size greater than 10" if sqs_polling_amount > 10
         end
 
-        # Sets a flag that instructs the publisher to reset the connection the next time it's used
+        # Resets the API client connection and provides @@reset_at so we know when the last time that was done
         def self.reset_connection!
           @@reset_at = Time.now
         end
 
         # Begins requesting messages from SQS, which will invoke the +&handler+ over each message
+        #
+        # @param [Block] &handler Message handler, used by the calling context (worker) to create & assigns a UnitOfWork
+        #
+        # @return [Array<Aws::SQS::Message>]
         def consume(&handler)
           while running?
             begin
               messages = handle_messages(&handler)
               sleep (Chore.config.consumer_sleep_interval) if messages.empty?
-            rescue AWS::SQS::Errors::NonExistentQueue => e
+            rescue Aws::SQS::Errors::NonExistentQueue => e
               Chore.logger.error "You specified a queue '#{queue_name}' that does not exist. You must create the queue before starting Chore. Shutting down..."
               raise Chore::TerribleMistake
             rescue => e
@@ -43,21 +45,34 @@ module Chore
           end
         end
 
-        # Rejects the given message from SQS by +id+. Currently a noop
-        def reject(id)
-
+        # Unimplemented. Rejects the given message from SQS.
+        #
+        # @param [String] message_id Unique ID of the SQS message
+        #
+        # @return nil
+        def reject(message_id)
         end
 
-        # Deletes the given message from SQS by +id+
-        def complete(id)
-          Chore.logger.debug "Completing (deleting): #{id}"
-          queue.batch_delete([id])
+        # Deletes the given message from the SQS queue
+        #
+        # @param [String] message_id Unique ID of the SQS message
+        # @param [Hash] receipt_handle Receipt handle (unique per consume request) of the SQS message
+        def complete(message_id, receipt_handle)
+          Chore.logger.debug "Completing (deleting): #{message_id}"
+          queue.delete_messages(entries: [{ id: message_id, receipt_handle: receipt_handle }])
         end
 
+        # Delays retry of a job by +backoff_calc+ seconds.
+        #
+        # @param [UnitOfWork] item Item to be delayed
+        # @param [Block] backoff_calc Code that determines the backoff.
         def delay(item, backoff_calc)
           delay = backoff_calc.call(item)
           Chore.logger.debug "Delaying #{item.id} by #{delay} seconds"
-          queue.batch_change_visibility(delay, [item.id])
+
+          queue.change_message_visibility_batch(entries: [
+            { id: item.id, receipt_handle: item.receipt_handle, visibility_timeout: delay },
+          ])
 
           return delay
         end
@@ -66,46 +81,59 @@ module Chore
 
         # Requests messages from SQS, and invokes the provided +&block+ over each one. Afterwards, the :on_fetch
         # hook will be invoked, per message
+        #
+        # @param [Block] &handler Message handler, passed along by #consume
+        #
+        # @return [Array<Aws::SQS::Message>]
         def handle_messages(&block)
-          msg = queue.receive_messages(:limit => sqs_polling_amount, :attributes => [:receive_count])
+          msg = queue.receive_messages(:max_number_of_messages => sqs_polling_amount, :attribute_names => ['ApproximateReceiveCount'])
           messages = *msg
+
           messages.each do |message|
-            unless duplicate_message?(message.id, message.queue.url, queue_timeout)
-              block.call(message.handle, queue_name, queue_timeout, message.body, message.receive_count - 1)
+            unless duplicate_message?(message.message_id, message.queue_url, queue_timeout)
+              block.call(message.message_id, message.receipt_handle, queue_name, queue_timeout, message.body, message.attributes['ApproximateReceiveCount'].to_i - 1)
             end
-            Chore.run_hooks_for(:on_fetch, message.handle, message.body)
+            Chore.run_hooks_for(:on_fetch, message.receipt_handle, message.body)
           end
+
           messages
         end
 
-        # Retrieves the SQS queue with the given +name+. The method will cache the results to prevent round trips on
-        # subsequent calls. If <tt>reset_connection!</tt> has been called, this will result in the connection being
-        # re-initialized, as well as clear any cached results from prior calls
+        # Retrieves the SQS queue object. The method will cache the results to prevent round trips on subsequent calls
+        #
+        # If <tt>reset_connection!</tt> has been called, this will result in the connection being re-initialized,
+        # as well as clear any cached results from prior calls
+        #
+        # @return [Aws::SQS::Queue]
         def queue
           if !@sqs_last_connected || (@@reset_at && @@reset_at >= @sqs_last_connected)
-            AWS::Core::Http::ConnectionPool.pools.each do |p|
-              p.empty!
-            end
+            Aws.empty_connection_pools!
             @sqs = nil
             @sqs_last_connected = Time.now
             @queue = nil
           end
-          @queue_url ||= sqs.queues.url_for(@queue_name)
-          @queue ||= sqs.queues[@queue_url]
+
+          @queue_url ||= sqs.get_queue_url(queue_name: @queue_name).queue_url
+          @queue ||= Aws::SQS::Queue.new(url: @queue_url, client: sqs)
         end
 
-        # The visibility timeout of the queue for this consumer
+        # The visibility timeout (in seconds) of the queue
+        #
+        # @return [Integer]
         def queue_timeout
-          @queue_timeout ||= queue.visibility_timeout
+          @queue_timeout ||= queue.attributes['VisibilityTimeout'].to_i
         end
 
-        # Access to the configured SQS connection object
+        # SQS API client object
+        #
+        # @return [Aws::SQS::Client]
         def sqs
-          @sqs ||= AWS::SQS.new(
-            :access_key_id => Chore.config.aws_access_key,
-            :secret_access_key => Chore.config.aws_secret_key)
+          @sqs ||= Chore::Queues::SQS.sqs_client
         end
 
+        # Maximum number of messages to retrieve on each request
+        #
+        # @return [Integer]
         def sqs_polling_amount
           Chore.config.queue_polling_size
         end