chore-core 1.10.0 → 4.0.0

data/lib/chore/queues/filesystem/consumer.rb

@@ -15,61 +15,111 @@ module Chore
       # Once complete job files are deleted.
       # If rejected they are moved back into new and will be processed again.  This may not be the
       # desired behavior long term and we may want to add configuration to this class to allow more
-      # creating failure handling and retrying. 
+      # creating failure handling and retrying.
       class Consumer < Chore::Consumer
         extend FilesystemQueue
 
         Chore::CLI.register_option 'fs_queue_root', '--fs-queue-root DIRECTORY', 'Root directory for fs based queue'
-        
-        FILE_QUEUE_MUTEXES = {}
 
         class << self
-          # Cleans up the in-progress files by making them new again.  This should only
-          # happen once per process.
-          def cleanup(queue)
-            new_dir = self.new_dir(queue)
-            in_progress_dir = self.in_progress_dir(queue)
-
-            job_files(in_progress_dir).each do |file|
-              make_new_again(file, new_dir, in_progress_dir)
+          # Cleans up expired in-progress files by making them new again.
+          def cleanup(expiration_time, new_dir, in_progress_dir)
+            each_file(in_progress_dir) do |job_file|
+              id, previous_attempts, timestamp = file_info(job_file)
+              next if timestamp > expiration_time
+
+              begin
+                make_new_again(job_file, new_dir, in_progress_dir)
+              rescue Errno::ENOENT
+                # File no longer exists; skip since it's been recovered by another
+                # consumer
+              rescue ArgumentError
+                # Move operation was attempted at same time as another consumer;
+                # skip since the other process succeeded where this one didn't
+              end
             end
           end
 
-          def make_in_progress(job, new_dir, in_progress_dir)
-            move_job(File.join(new_dir, job), File.join(in_progress_dir, job))
+          # Moves job file to inprogress directory and returns the full path
+          # if the job was successfully locked by this consumer
+          def make_in_progress(job, new_dir, in_progress_dir, queue_timeout)
+            basename, previous_attempts, * = file_info(job)
+
+            from = File.join(new_dir, job)
+            # Add a timestamp to mark when the job was started
+            to = File.join(in_progress_dir, "#{basename}.#{previous_attempts}.#{Time.now.to_i}.job")
+
+            # If the file is non-zero, this means it was successfully written to
+            # by a publisher and we can attempt to move it to "in progress".
+            #
+            # There is a small window of time where the file can be zero, but
+            # the publisher hasn't finished writing to the file yet.
+            if !File.zero?(from)
+              File.open(from, "r") do |f|
+                # If the lock can't be obtained, that means it's been locked
+                # by another consumer or the publisher of the file) -- don't
+                # block and skip it
+                if f.flock(File::LOCK_EX | File::LOCK_NB)
+                  FileUtils.mv(from, to)
+                  to
+                end
+              end
+            elsif (Time.now - File.ctime(from)) >= queue_timeout
+              # The file is empty (zero bytes) and enough time has passed since
+              # the file was written that we can safely assume it will never
+              # get written to be the publisher.
+              #
+              # The scenario where this happens is when the publisher created
+              # the file, but the process was killed before it had a chance to
+              # actually write the data.
+              File.delete(from)
+              nil
+            end
+          rescue Errno::ENOENT
+            # File no longer exists; skip it since it's been picked up by
+            # another consumer
           end
 
+          # Moves job file to new directory and returns the full path
           def make_new_again(job, new_dir, in_progress_dir)
             basename, previous_attempts = file_info(job)
-            move_job(File.join(in_progress_dir, job), File.join(new_dir, "#{basename}.#{previous_attempts + 1}.job"))
-          end
 
-          # Moves job file to inprogress directory and returns the full path
-          def move_job(from, to)
-            f = File.open(from, "r")
-            # wait on the lock a publisher in another process might have.
-            # Once we get the lock the file is ours to move to mark it in progress
-            f.flock(File::LOCK_EX)
-            begin
-              FileUtils.mv(f.path, to)
-            ensure
-              f.flock(File::LOCK_UN) # yes we can unlock it after its been moved, I checked
-            end
+            from = File.join(in_progress_dir, job)
+            to = File.join(new_dir, "#{basename}.#{previous_attempts + 1}.job")
+            FileUtils.mv(from, to)
+
             to
           end
 
-          def job_files(dir)
-            Dir.entries(dir).select{|e| ! e.start_with?(".")}
+          def each_file(path, limit = nil)
+            count = 0
+
+            Dir.foreach(path) do |file|
+              next if file.start_with?('.')
+
+              yield file
+
+              count += 1
+              break if limit && count >= limit
+            end
           end
 
           # Grabs the unique identifier for the job filename and the number of times
           # it's been attempted (also based on the filename)
           def file_info(job_file)
-            id, previous_attempts = File.basename(job_file, '.job').split('.')
-            [id, previous_attempts.to_i]
+            id, previous_attempts, timestamp, * = job_file.split('.')
+            [id, previous_attempts.to_i, timestamp.to_i]
           end
         end
 
+        # The minimum number of seconds to allow to pass between checks for expired
+        # jobs on the filesystem.
+        #
+        # Since queue times are measured on the order of seconds, 1 second is the
+        # smallest duration.  It also prevents us from burning a lot of CPU looking
+        # at expired jobs when the consumer sleep interval is less than 1 second.
+        EXPIRATION_CHECK_INTERVAL = 1
+
         # The amount of time units of work can run before the queue considers
         # them timed out.  For filesystem queues, this is the global default.
         attr_reader :queue_timeout
@@ -77,66 +127,79 @@ module Chore
         def initialize(queue_name, opts={})
           super(queue_name, opts)
 
-          # Even though putting these Mutexes in this hash is, by itself, not particularly threadsafe
-          # as long as some Mutex ends up in the queue after all consumers are created we're good
-          # as they are pulled from the queue and synchronized for file operations below
-          FILE_QUEUE_MUTEXES[@queue_name] ||= Mutex.new
-
           @in_progress_dir = self.class.in_progress_dir(queue_name)
           @new_dir = self.class.new_dir(queue_name)
-          @queue_timeout = Chore.config.default_queue_timeout
+          @queue_timeout = self.class.queue_timeout(queue_name)
         end
 
-        def consume(&handler)
+        def consume
           Chore.logger.info "Starting consuming file system queue #{@queue_name} in #{self.class.queue_dir(queue_name)}"
           while running?
             begin
-              #TODO move expired job files to new directory?
-              handle_jobs(&handler)
+              # Move expired job files to new directory (so long as enough time has
+              # passed since we last did this check)
+              if !@last_cleaned_at || (Time.now - @last_cleaned_at).to_i >= EXPIRATION_CHECK_INTERVAL
+                self.class.cleanup(Time.now.to_i - @queue_timeout, @new_dir, @in_progress_dir)
+                @last_cleaned_at = Time.now
+              end
+
+              found_files = false
+              handle_messages do |*args|
+                found_files = true
+                yield(*args)
+              end
             rescue => e
               Chore.logger.error { "#{self.class}#consume: #{e} #{e.backtrace * "\n"}" }
             ensure
-              sleep 5
+              sleep(Chore.config.consumer_sleep_interval) unless found_files
             end
           end
         end
 
+        # Rejects the given message from the filesystem by +id+. Currently a noop
         def reject(id)
-          Chore.logger.debug "Rejecting: #{id}"
-          make_new_again(id)
+
         end
 
-        def complete(id)
-          Chore.logger.debug "Completing (deleting): #{id}"
-          FileUtils.rm_f(File.join(@in_progress_dir, id))
+        # Deletes the given message from filesystem queue. Since the filesystem is not a remote API, there is no
+        # notion of a "receipt handle".
+        #
+        # @param [String] message_id Unique ID of the message
+        # @param [Hash] receipt_handle Receipt handle of the message. Always nil for the filesystem consumer
+        def complete(message_id, receipt_handle = nil)
+          Chore.logger.debug "Completing (deleting): #{message_id}"
+          File.delete(File.join(@in_progress_dir, message_id))
+        rescue Errno::ENOENT
+          # The job took too long to complete, was deemed expired, and moved
+          # back into "new".  Ignore.
         end
 
         private
 
         # finds all new job files, moves them to in progress and starts the job
         # Returns a list of the job files processed
-        def handle_jobs(&block)
-          # all consumers on a single queue share a lock on handling files.
-          # Each consumer comes along, processes all present files and release the lock.
-          # This isn't particularly useful but is here to allow the configuration of
-          # ThreadedConsumerStrategy with mutiple threads on a queue safely although you
-          # probably wouldn't want to do that.
-          FILE_QUEUE_MUTEXES[@queue_name].synchronize do
-            self.class.job_files(@new_dir).each do |job_file|
-              Chore.logger.debug "Found a new job #{job_file}"
-              
-              job_json = File.read(make_in_progress(job_file))
-              basename, previous_attempts = self.class.file_info(job_file)
-
-              # job_file is just the name which is the job id
-              block.call(job_file, queue_name, queue_timeout, job_json, previous_attempts)
-              Chore.run_hooks_for(:on_fetch, job_file, job_json)
-            end
+        def handle_messages(&block)
+          self.class.each_file(@new_dir, Chore.config.queue_polling_size) do |job_file|
+            Chore.logger.debug "Found a new job #{job_file}"
+
+            in_progress_path = make_in_progress(job_file)
+            next unless in_progress_path
+
+            # The job filename may have changed, so update it to reflect the in progress path
+            job_file = File.basename(in_progress_path)
+
+            job_json = File.read(in_progress_path)
+            basename, previous_attempts, * = self.class.file_info(job_file)
+
+            # job_file is just the name which is the job id. 2nd argument (:receipt_handle) is nil because the
+            # filesystem is dealt with directly, as opposed to being an external API
+            block.call(job_file, nil, queue_name, queue_timeout, job_json, previous_attempts)
+            Chore.run_hooks_for(:on_fetch, job_file, job_json)
           end
         end
 
         def make_in_progress(job)
-          self.class.make_in_progress(job, @new_dir, @in_progress_dir)
+          self.class.make_in_progress(job, @new_dir, @in_progress_dir, @queue_timeout)
         end
 
         def make_new_again(job)
@@ -146,4 +209,3 @@ module Chore
     end
   end
 end
-

data/lib/chore/queues/filesystem/filesystem_queue.rb

@@ -6,6 +6,8 @@ module Chore::FilesystemQueue
   NEW_JOB_DIR = "new"
   # Local directory for jobs currently in-process to be moved
   IN_PROGRESS_DIR = "inprogress"
+  # Local directory for configuration info
+  CONFIG_DIR = "config"
   
   # Retrieves the directory for in-process messages to go. If the directory for the +queue_name+ doesn't exist,
   # it will be created for you. If the directory cannot be created, an IOError will be raised
@@ -29,6 +31,23 @@ module Chore::FilesystemQueue
     prepare_dir(File.join(root_dir, queue_name))
   end
 
+  # The configuration for the given queue
+  def config_dir(queue_name)
+    validate_dir(queue_name, CONFIG_DIR)
+  end
+
+  def config_value(queue_name, config_name)
+    config_file = File.join(config_dir(queue_name), config_name)
+    if File.exists?(config_file)
+      File.read(config_file).strip
+    end
+  end
+
+  # Returns the timeout for +queue_name+
+  def queue_timeout(queue_name)
+    (config_value(queue_name, 'timeout') || Chore.config.default_queue_timeout).to_i
+  end
+
   private
   # Returns the directory for the given +queue_name+ and +task_state+. If the directory doesn't exist, it will be
   # created for you. If the directory cannot be created, an IOError will be raised

data/lib/chore/queues/filesystem/publisher.rb

@@ -10,27 +10,20 @@ module Chore
         # See the top of FilesystemConsumer for comments on how this works
         include FilesystemQueue
 
-        # Mutex for holding a lock over the files for this queue while they are in process
-        FILE_MUTEX = Mutex.new
-
         # use of mutex and file locking should make this both threadsafe and safe for multiple
-        # processes to use the same queue directory simultaneously. 
+        # processes to use the same queue directory simultaneously.
         def publish(queue_name,job)
           # First try encoding the job to avoid writing empty job files if this fails
           encoded_job = encode_job(job)
 
-          FILE_MUTEX.synchronize do
-            while true
-              # keep trying to get a file with nothing in it meaning we just created it
-              # as opposed to us getting someone else's file that hasn't been processed yet.
-              f = File.open(filename(queue_name, job[:class].to_s), "w")
+          published = false
+          while !published
+            # keep trying to get a file with nothing in it meaning we just created it
+            # as opposed to us getting someone else's file that hasn't been processed yet.
+            File.open(filename(queue_name, job[:class].to_s), "a") do |f|
               if f.flock(File::LOCK_EX | File::LOCK_NB) && f.size == 0
-                begin
-                  f.write(encoded_job)
-                ensure
-                  f.flock(File::LOCK_UN)
-                end
-                break
+                f.write(encoded_job)
+                published = true
               end
             end
           end
@@ -40,7 +33,8 @@ module Chore
         def filename(queue_name, job_name)
           now = Time.now.strftime "%Y%m%d-%H%M%S-%6N"
           previous_attempts = 0
-          File.join(new_dir(queue_name), "#{queue_name}-#{job_name}-#{now}.#{previous_attempts}.job")
+          pid = Process.pid
+          File.join(new_dir(queue_name), "#{queue_name}-#{job_name}-#{pid}-#{now}.#{previous_attempts}.job")
         end
       end
     end

data/lib/chore/queues/sqs.rb

@@ -1,10 +1,18 @@
 module Chore
   module Queues
     module SQS
+      def self.sqs_client
+        Aws::SQS::Client.new(logger: Chore.logger, log_level: Chore.log_level_to_sym)
+      end
+
       # Helper method to create queues based on the currently known list as provided by your configured Chore::Jobs
       # This is meant to be invoked from a rake task, and not directly.
       # These queues will be created with the default settings, which may not be ideal.
       # This is meant only as a convenience helper for testing, and not as a way to create production quality queues in SQS
+      #
+      # @param [TrueClass, FalseClass] halt_on_existing Raise an exception if the queue already exists
+      #
+      # @return [Array<String>]
       def self.create_queues!(halt_on_existing=false)
         raise 'You must have atleast one Chore Job configured and loaded before attempting to create queues' unless Chore.prefixed_queue_names.length > 0
 
@@ -20,49 +28,50 @@ module Chore
           end
         end
 
-        #This will raise an exception if AWS has not been configured by the project making use of Chore
-        sqs_queues = AWS::SQS.new.queues
         Chore.prefixed_queue_names.each do |queue_name|
           Chore.logger.info "Chore Creating Queue: #{queue_name}"
           begin
-            sqs_queues.create(queue_name)
-          rescue AWS::SQS::Errors::QueueAlreadyExists
+            sqs_client.create_queue(queue_name: queue_name)
+          rescue Aws::SQS::Errors::QueueAlreadyExists
             Chore.logger.info "exists with different config"
           end
         end
+
         Chore.prefixed_queue_names
       end
 
       # Helper method to delete all known queues based on the list as provided by your configured Chore::Jobs
       # This is meant to be invoked from a rake task, and not directly.
+      #
+      # @return [Array<String>]
+
       def self.delete_queues!
         raise 'You must have atleast one Chore Job configured and loaded before attempting to create queues' unless Chore.prefixed_queue_names.length > 0
-        #This will raise an exception if AWS has not been configured by the project making use of Chore
-        sqs_queues = AWS::SQS.new.queues
+
         Chore.prefixed_queue_names.each do |queue_name|
           begin
             Chore.logger.info "Chore Deleting Queue: #{queue_name}"
-            url = sqs_queues.url_for(queue_name)
-            sqs_queues[url].delete
+            url = sqs_client.get_queue_url(queue_name: queue_name).queue_url
+            sqs_client.delete_queue(queue_url: url)
           rescue => e
             # This could fail for a few reasons - log out why it failed, then continue on
             Chore.logger.error "Deleting Queue: #{queue_name} failed because #{e}"
           end
         end
+
         Chore.prefixed_queue_names
       end
 
       # Collect a list of queues that already exist
+      #
+      # @return [Array<String>]
       def self.existing_queues
-        #This will raise an exception if AWS has not been configured by the project making use of Chore
-        sqs_queues = AWS::SQS.new.queues
-
         Chore.prefixed_queue_names.select do |queue_name|
           # If the NonExistentQueue exception is raised we do not care about that queue name.
           begin
-            sqs_queues.named(queue_name)
+            sqs_client.get_queue_url(queue_name: queue_name)
             true
-          rescue AWS::SQS::Errors::NonExistentQueue
+          rescue Aws::SQS::Errors::NonExistentQueue
             false
           end
         end

data/lib/chore/queues/sqs/consumer.rb

@@ -1,9 +1,6 @@
-require 'aws/sqs'
+require 'aws-sdk-sqs'
 require 'chore/duplicate_detector'
 
-AWS.eager_autoload! AWS::Core
-AWS.eager_autoload! AWS::SQS
-
 module Chore
   module Queues
     module SQS
@@ -16,26 +13,30 @@ module Chore
         Chore::CLI.register_option 'aws_access_key', '--aws-access-key KEY', 'Valid AWS Access Key'
         Chore::CLI.register_option 'aws_secret_key', '--aws-secret-key KEY', 'Valid AWS Secret Key'
         Chore::CLI.register_option 'dedupe_servers', '--dedupe-servers SERVERS', 'List of mememcache compatible server(s) to use for storing SQS Message Dedupe cache'
-        Chore::CLI.register_option 'queue_polling_size', '--queue_polling_size NUM', Integer, 'Amount of messages to grab on each request' do |arg|
-          raise ArgumentError, "Cannot specify a queue polling size greater than 10" if arg > 10
-        end
 
+        # @param [String] queue_name Name of SQS queue
+        # @param [Hash] opts Options
         def initialize(queue_name, opts={})
           super(queue_name, opts)
+          raise Chore::TerribleMistake, "Cannot specify a queue polling size greater than 10" if sqs_polling_amount > 10
         end
 
-        # Sets a flag that instructs the publisher to reset the connection the next time it's used
+        # Resets the API client connection and provides @@reset_at so we know when the last time that was done
         def self.reset_connection!
           @@reset_at = Time.now
         end
 
         # Begins requesting messages from SQS, which will invoke the +&handler+ over each message
+        #
+        # @param [Block] &handler Message handler, used by the calling context (worker) to create & assigns a UnitOfWork
+        #
+        # @return [Array<Aws::SQS::Message>]
         def consume(&handler)
           while running?
             begin
               messages = handle_messages(&handler)
-              sleep (Chore.config.consumer_sleep_interval || 1) if messages.empty?
-            rescue AWS::SQS::Errors::NonExistentQueue => e
+              sleep (Chore.config.consumer_sleep_interval) if messages.empty?
+            rescue Aws::SQS::Errors::NonExistentQueue => e
               Chore.logger.error "You specified a queue '#{queue_name}' that does not exist. You must create the queue before starting Chore. Shutting down..."
               raise Chore::TerribleMistake
             rescue => e
@@ -44,21 +45,34 @@ module Chore
           end
         end
 
-        # Rejects the given message from SQS by +id+. Currently a noop
-        def reject(id)
-
+        # Unimplemented. Rejects the given message from SQS.
+        #
+        # @param [String] message_id Unique ID of the SQS message
+        #
+        # @return nil
+        def reject(message_id)
         end
 
-        # Deletes the given message from SQS by +id+
-        def complete(id)
-          Chore.logger.debug "Completing (deleting): #{id}"
-          queue.batch_delete([id])
+        # Deletes the given message from the SQS queue
+        #
+        # @param [String] message_id Unique ID of the SQS message
+        # @param [Hash] receipt_handle Receipt handle (unique per consume request) of the SQS message
+        def complete(message_id, receipt_handle)
+          Chore.logger.debug "Completing (deleting): #{message_id}"
+          queue.delete_messages(entries: [{ id: message_id, receipt_handle: receipt_handle }])
         end
 
+        # Delays retry of a job by +backoff_calc+ seconds.
+        #
+        # @param [UnitOfWork] item Item to be delayed
+        # @param [Block] backoff_calc Code that determines the backoff.
         def delay(item, backoff_calc)
           delay = backoff_calc.call(item)
           Chore.logger.debug "Delaying #{item.id} by #{delay} seconds"
-          queue.batch_change_visibility(delay, [item.id])
+
+          queue.change_message_visibility_batch(entries: [
+            { id: item.id, receipt_handle: item.receipt_handle, visibility_timeout: delay },
+          ])
 
           return delay
         end
@@ -67,62 +81,61 @@ module Chore
 
         # Requests messages from SQS, and invokes the provided +&block+ over each one. Afterwards, the :on_fetch
         # hook will be invoked, per message
+        #
+        # @param [Block] &handler Message handler, passed along by #consume
+        #
+        # @return [Array<Aws::SQS::Message>]
         def handle_messages(&block)
-          msg = queue.receive_messages(:limit => sqs_polling_amount, :attributes => [:receive_count])
+          msg = queue.receive_messages(:max_number_of_messages => sqs_polling_amount, :attribute_names => ['ApproximateReceiveCount'])
           messages = *msg
+
           messages.each do |message|
-            unless duplicate_message?(message)
-              block.call(message.handle, queue_name, queue_timeout, message.body, message.receive_count - 1)
+            unless duplicate_message?(message.message_id, message.queue_url, queue_timeout)
+              block.call(message.message_id, message.receipt_handle, queue_name, queue_timeout, message.body, message.attributes['ApproximateReceiveCount'].to_i - 1)
             end
-            Chore.run_hooks_for(:on_fetch, message.handle, message.body)
+            Chore.run_hooks_for(:on_fetch, message.receipt_handle, message.body)
           end
-          messages
-        end
-
-        # Checks if the given message has already been received within the timeout window for this queue
-        def duplicate_message?(message)
-          dupe_detector.found_duplicate?(:id=>message.id, :queue=>message.queue.url, :visibility_timeout=>message.queue.visibility_timeout)
-        end
 
-        # Returns the instance of the DuplicateDetector used to ensure unique messages.
-        # Will create one if one doesn't already exist
-        def dupe_detector
-          @dupes ||= DuplicateDetector.new({:servers => Chore.config.dedupe_servers,
-                                            :dupe_on_cache_failure => Chore.config.dupe_on_cache_failure})
+          messages
         end
 
-        # Retrieves the SQS queue with the given +name+. The method will cache the results to prevent round trips on
-        # subsequent calls. If <tt>reset_connection!</tt> has been called, this will result in the connection being
-        # re-initialized, as well as clear any cached results from prior calls
+        # Retrieves the SQS queue object. The method will cache the results to prevent round trips on subsequent calls
+        #
+        # If <tt>reset_connection!</tt> has been called, this will result in the connection being re-initialized,
+        # as well as clear any cached results from prior calls
+        #
+        # @return [Aws::SQS::Queue]
         def queue
           if !@sqs_last_connected || (@@reset_at && @@reset_at >= @sqs_last_connected)
-            AWS::Core::Http::ConnectionPool.pools.each do |p|
-              p.empty!
-            end
+            Aws.empty_connection_pools!
             @sqs = nil
             @sqs_last_connected = Time.now
             @queue = nil
           end
-          @queue_url ||= sqs.queues.url_for(@queue_name)
-          @queue ||= sqs.queues[@queue_url]
+
+          @queue_url ||= sqs.get_queue_url(queue_name: @queue_name).queue_url
+          @queue ||= Aws::SQS::Queue.new(url: @queue_url, client: sqs)
         end
 
-        # The visibility timeout of the queue for this consumer
+        # The visibility timeout (in seconds) of the queue
+        #
+        # @return [Integer]
         def queue_timeout
-          @queue_timeout ||= queue.visibility_timeout
+          @queue_timeout ||= queue.attributes['VisibilityTimeout'].to_i
         end
 
-        # Access to the configured SQS connection object
+        # SQS API client object
+        #
+        # @return [Aws::SQS::Client]
         def sqs
-          @sqs ||= AWS::SQS.new(
-            :access_key_id => Chore.config.aws_access_key,
-            :secret_access_key => Chore.config.aws_secret_key,
-            :logger => Chore.logger,
-            :log_level => :debug)
+          @sqs ||= Chore::Queues::SQS.sqs_client
         end
 
+        # Maximum number of messages to retrieve on each request
+        #
+        # @return [Integer]
         def sqs_polling_amount
-          Chore.config.queue_polling_size || 10
+          Chore.config.queue_polling_size
         end
       end
     end