chore-core 1.8.2 → 3.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f1a706f2f8f8eefe36f16cfaea8636f6fe13df8d
-  data.tar.gz: d4c0b0f97c85916e509ec609bd9a9e38cac6aac0
+SHA256:
+  metadata.gz: 5aeebbb3efb30c9047c67996272864003542d9bd91d9887d3e149b6840f0573a
+  data.tar.gz: dd210d2eda88ddfab6e97646cb69ecccfc01a57a4198a8c2fc50c30d260dd32f
 SHA512:
-  metadata.gz: c92dcbc42098e17b5e1d3ac05860ca1b74022725a2839964f6480bc8c3f080555e950757997ad9d3d4cdbd4b122e02e467614bb481a4108dfe65c5a74b797c6c
-  data.tar.gz: 95baf93fe723268b499447c24464443e3e7e19ed62d5718394c7dae66bcafb44a33f47133b3973827efd5856acc4fee2aab795adc33515c18068807f9e5039aa
+  metadata.gz: 0df90580322b71a356a1b7bed8d1480c0794e6fcd12995bc82edfb3a878f40981fb87fefdcb2ad97a9841cabad2a151268a0e098d08f697cd56b727fcb973948
+  data.tar.gz: 24ef7ff568162c5f173ef599d687cd768c9b8a85948d5c80df10855fe30eac9df118e6a15b741e348e064e51ab693b395d81bf4e3689ca9519d9c15b69d05521

data/README.md

@@ -1,5 +1,7 @@
 # Chore: Job processing... for the future!
 
+[![Build Status](https://travis-ci.org/Tapjoy/chore.svg?branch=master)](https://travis-ci.org/Tapjoy/chore)
+
 ## About
 
 Chore is a pluggable, multi-backend job processor. It was built from the ground up to be extremely flexible. We hope that you
@@ -33,6 +35,7 @@ Other options include:
     --threads-per-queue 4 # number of threads per queue for consuming from a given queue.
     --dedupe-servers # if using SQS or similiar queue with at-least once delivery and your memcache is running on something other than localhost
     --batch-size 50 # how many messages are batched together before handing them to a worker
+    --batch-timeout 20 # maximum number of seconds to wait until handing a message over to a worker
     --queue_prefix prefixy # A prefix to prepend to queue names, mainly for development and qa testing purposes
     --max-attempts 100 # The maximum number of times a job can be attempted
     --dupe-on-cache-failure # Determines the deduping behavior when a cache connection error occurs. When set to `false`, the message is assumed not to be a duplicate. Defaults to `false`.
@@ -92,6 +95,7 @@ Chore.configure do |c|
   c.max_attempts = 100
   ...
   c.batch_size = 50
+  c.batch_timeout = 20
 end
 ```
 
@@ -189,11 +193,13 @@ A number of hooks, both global and per-job, exist in Chore for your convenience.
 
 Global Hooks:
 
+* before_start
 * before_first_fork
 * before_fork
 * after_fork
 * around_fork
 * within_fork
+* before_shutdown
 
 ("within_fork" behaves similarly to around_fork, except that it is called after the worker process has been forked. In contrast, around_fork is called by the parent process.)
 

data/chore-core.gemspec

@@ -39,6 +39,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency(%q<json>, [">= 0"])
   s.add_runtime_dependency(%q<aws-sdk-v1>, ["~> 1.56", ">= 1.56.0"])
   s.add_runtime_dependency(%q<thread>, ["~> 0.1.3"])
+  s.add_runtime_dependency('get_process_mem', ["~> 0.2.0"])
   s.add_development_dependency(%q<rspec>, ["~> 3.3.0"])
   s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
   s.add_development_dependency(%q<bundler>, [">= 0"])

data/lib/chore.rb

@@ -34,13 +34,18 @@ module Chore #:nodoc:
     :fetcher               => Fetcher,
     :consumer_strategy     => Strategy::ThreadedConsumerStrategy,
     :batch_size            => 50,
+    :batch_timeout         => 20,
     :log_level             => Logger::WARN,
     :log_path              => STDOUT,
     :default_queue_timeout => (12 * 60 * 60), # 12 hours
     :shutdown_timeout      => (2 * 60),
     :max_attempts          => 1.0 / 0.0, # Infinity
     :dupe_on_cache_failure => false,
-    :payload_handler => Chore::Job
+    :queue_polling_size    => 10,
+    :payload_handler       => Chore::Job,
+    :master_procline       => "chore-master-#{Chore::VERSION}",
+    :worker_procline       => "chore-worker-#{Chore::VERSION}",
+    :consumer_sleep_interval => 1
   }
 
   class << self
@@ -110,9 +115,9 @@ module Chore #:nodoc:
   #   add_hook(:before_fork) {|worker| puts 1 }
   #   add_hook(:before_fork) {|worker| puts 2 }
   #   add_hook(:before_fork) {|worker| puts 3 }
-  #   
+  #
   #   run_hooks_for(:before_fork, worker)
-  #   
+  #
   #   # ...will produce the following output
   #   => 1
   #   => 2
@@ -129,9 +134,9 @@ module Chore #:nodoc:
   #   add_hook(:around_fork) {|worker, &block| puts 'before 1'; block.call; puts 'after 1'}
   #   add_hook(:around_fork) {|worker, &block| puts 'before 2'; block.call; puts 'after 2'}
   #   add_hook(:around_fork) {|worker, &block| puts 'before 3'; block.call; puts 'after 3'}
-  #   
+  #
   #   run_hooks_for(:around_fork, worker) { puts 'block' }
-  #   
+  #
   #   # ...will produce the following output
   #   => before 1
   #   => before 2
@@ -186,6 +191,7 @@ module Chore #:nodoc:
   #   Chore.configure do |c|
   #     c.consumer = Chore::Queues::SQS::Consumer
   #     c.batch_size = 50
+  #     c.batch_timeout = 20
   #   end
   def self.configure(opts={})
     @config = (@config ? @config.merge_hash(opts) : Chore::Configuration.new(DEFAULT_OPTIONS.merge(opts)))

data/lib/chore/cli.rb

@@ -89,6 +89,7 @@ module Chore #:nodoc:
       detect_queues
       Chore.configure(options)
       Chore.configuring = false
+      validate_strategy!
     end
 
 
@@ -134,7 +135,7 @@ module Chore #:nodoc:
         options[:consumer_strategy] = constantize(arg)
       end
 
-      register_option 'consumer_sleep_interval', '--consumer-sleep-interval INTERVAL', Float, 'Length of time in seconds to sleep when the consumer does not find any messages. Defaults vary depending on consumer implementation'
+      register_option 'consumer_sleep_interval', '--consumer-sleep-interval INTERVAL', Float, 'Length of time in seconds to sleep when the consumer does not find any messages (default: 1)'
 
       register_option 'payload_handler', '--payload_handler CLASS_NAME', 'Name of a class to use as the payload handler (default: Chore::Job)' do |arg|
         options[:payload_handler] = constantize(arg)
@@ -144,6 +145,7 @@ module Chore #:nodoc:
 
       register_option 'dupe_on_cache_failure', '--dupe-on-cache-failure BOOLEAN', 'Determines the deduping behavior when a cache connection error occurs. When set to false, the message is assumed not to be a duplicate. (default: false)'
 
+      register_option 'queue_polling_size', '--queue_polling_size NUM', Integer, 'Amount of messages to grab on each request (default: 10)'
     end
 
     def parse_opts(argv, ignore_errors = false) #:nodoc:
@@ -254,7 +256,6 @@ module Chore #:nodoc:
     end
 
     def validate! #:nodoc:
-
       missing_option!("--require [PATH|DIR]") unless options[:require]
 
       if !File.exist?(options[:require]) ||
@@ -266,7 +267,25 @@ module Chore #:nodoc:
         puts @parser
         exit(1)
       end
+    end
 
+    def validate_strategy!
+      consumer_strategy = Chore.config.consumer_strategy.to_s
+      worker_strategy = Chore.config.worker_strategy.to_s
+
+      throttled_consumer = 'Chore::Strategy::ThrottledConsumerStrategy'
+      preforked_worker = 'Chore::Strategy::PreForkedWorkerStrategy'
+
+      if consumer_strategy == throttled_consumer || worker_strategy == preforked_worker
+        unless consumer_strategy == throttled_consumer && worker_strategy == preforked_worker
+          puts "=================================================================="
+          puts "  PreForkedWorkerStrategy may only be paired with   "
+          puts "  ThrottledConsumerStrategy or vice versa  "
+          puts "  Please check your configurations "
+          puts "=================================================================="
+          exit(1)
+        end
+      end
     end
   end
 end

data/lib/chore/consumer.rb

@@ -21,11 +21,6 @@ module Chore
     def self.reset_connection!
     end
 
-    # Cleans up any resources that were left behind from prior instances of the
-    # chore process.  By default, this is a no-op.
-    def self.cleanup(queue)
-    end
-
     # Consume takes a block with an arity of two. The two params are
     # |message_id,message_body| where message_id is any object that the
     # consumer will need to be able to act on a message later (reject, complete, etc)
@@ -53,5 +48,20 @@ module Chore
     def running?
       @running
     end
+
+    # returns up to n work
+    def provide_work(n)
+      raise NotImplementedError
+    end
+
+    # now, given an arbitrary key and klass, have we seen the key already?
+    def duplicate_message?(dedupe_key, klass, queue_timeout)
+      dupe_detector.found_duplicate?(:id=>dedupe_key, :queue=>klass.to_s, :visibility_timeout=>queue_timeout)
+    end
+
+    def dupe_detector
+      @dupes ||= DuplicateDetector.new({:servers => Chore.config.dedupe_servers,
+                                        :dupe_on_cache_failure => false})
+    end
   end
 end

data/lib/chore/fetcher.rb

@@ -11,21 +11,16 @@ module Chore
     # Starts the fetcher with the configured Consumer Strategy. This will begin consuming messages from your queue
     def start
       Chore.logger.info "Fetcher starting up"
-
-      # Clean up configured queues in case there are any resources left behind
-      Chore.config.queues.each do |queue|
-        Chore.config.consumer.cleanup(queue)
-      end
-
       @strategy.fetch
     end
 
     # Stops the fetcher, preventing any further messages from being pulled from the queue
     def stop!
       unless @stopping
-        Chore.logger.info "Fetcher shutting down"
+        Chore.logger.info "Fetcher shutting down started"
         @stopping = true
         @strategy.stop!
+        Chore.logger.info "Fetcher shutting down completed"
       end
     end
 
@@ -33,5 +28,15 @@ module Chore
     def stopping?
       @stopping
     end
+
+    # returns upto n work units
+    def provide_work(n)
+      @strategy.provide_work(n)
+    end
+
+    # gives work back to the consumer in case it couldn't be assigned
+    def return_work(work_units)
+      @strategy.return_work(work_units)
+    end
   end
 end

data/lib/chore/hooks.rb

@@ -15,7 +15,8 @@ module Chore
 
   private
     def hooks_for(event)
-      candidate_methods.grep(/^#{event}/).sort
+      @_chore_hooks ||= {}
+      @_chore_hooks[event] ||= candidate_methods.grep(/^#{event}/).sort
     end
 
     # NOTE: Any hook methods defined after this is first referenced (i.e.,

data/lib/chore/job.rb

@@ -60,6 +60,12 @@ module Chore
             raise ArgumentError, "#{self.to_s}: backoff must accept a single argument"
           end
         end
+
+        if @chore_options.key?(:dedupe_lambda)
+          if !@chore_options[:dedupe_lambda].is_a?(Proc)
+            raise ArgumentError, "#{self.to_s}: dedupe_lambda must be a lambda or Proc"
+          end
+        end
       end
 
       # This is a method so it can be overriden to create additional required
@@ -108,6 +114,17 @@ module Chore
       def has_backoff?
         self.options.key?(:backoff)
       end
+
+      def has_dedupe_lambda?
+        self.options.key?(:dedupe_lambda)
+      end
+
+      def dedupe_key(*args)
+        return unless has_dedupe_lambda?
+
+        # run the proc to get the key
+        self.options[:dedupe_lambda].call(*args).to_s
+      end
     end #ClassMethods
 
     # This is handy to override in an included job to be able to do job setup that requires

data/lib/chore/manager.rb

@@ -5,19 +5,21 @@ require 'chore/fetcher'
 module Chore
   # Manages the interactions between fetching messages (Consumer Strategy), and working over them (Worker Strategy)
   class Manager
+    include Util
 
     def initialize()
       Chore.logger.info "Booting Chore #{Chore::VERSION}"
       Chore.logger.debug { Chore.config.inspect }
+      procline("#{Chore.config.master_procline}:Started:#{Time.now}")
       @started_at = nil
       @worker_strategy = Chore.config.worker_strategy.new(self)
       @fetcher = Chore.config.fetcher.new(self)
-      @processed = 0
       @stopping = false
     end
 
     # Start the Manager. This calls both the #start method of the configured Worker Strategy, as well as Fetcher#start.
     def start
+      Chore.run_hooks_for(:before_start)
       @started_at = Time.now
       @worker_strategy.start
       @fetcher.start
@@ -26,11 +28,12 @@ module Chore
     # Shut down the Manager, the Worker Strategy, and the Fetcher. This calls the +:before_shutdown+ hook.
     def shutdown!
       unless @stopping
-        Chore.logger.info "Manager shutting down"
+        Chore.logger.info "Manager shutting down started"
         @stopping = true
         Chore.run_hooks_for(:before_shutdown)
         @fetcher.stop!
         @worker_strategy.stop!
+        Chore.logger.info "Manager shutting down completed"
       end
     end
 
@@ -41,7 +44,20 @@ module Chore
     # than they can be consumed.
     def assign(work)
       Chore.logger.debug { "Manager#assign: No. of UnitsOfWork: #{work.length})" }
+      work.each do | item |
+        Chore.run_hooks_for(:before_send_to_worker, item)
+      end
       @worker_strategy.assign(work) unless @stopping
     end
+
+    # returns up to n from the throttled consumer queue
+    def fetch_work(n)
+      @fetcher.provide_work(n)
+    end
+
+    # gives work back to the fetcher in case it couldn't be assigned
+    def return_work(work_units)
+      @fetcher.return_work(work_units)
+    end
   end
 end

data/lib/chore/queues/filesystem/consumer.rb

@@ -21,55 +21,105 @@ module Chore
 
         Chore::CLI.register_option 'fs_queue_root', '--fs-queue-root DIRECTORY', 'Root directory for fs based queue'
         
-        FILE_QUEUE_MUTEXES = {}
-
         class << self
-          # Cleans up the in-progress files by making them new again.  This should only
-          # happen once per process.
-          def cleanup(queue)
-            new_dir = self.new_dir(queue)
-            in_progress_dir = self.in_progress_dir(queue)
-
-            job_files(in_progress_dir).each do |file|
-              make_new_again(file, new_dir, in_progress_dir)
+          # Cleans up expired in-progress files by making them new again.
+          def cleanup(expiration_time, new_dir, in_progress_dir)
+            each_file(in_progress_dir) do |job_file|
+              id, previous_attempts, timestamp = file_info(job_file)
+              next if timestamp > expiration_time
+
+              begin
+                make_new_again(job_file, new_dir, in_progress_dir)
+              rescue Errno::ENOENT
+                # File no longer exists; skip since it's been recovered by another
+                # consumer
+              rescue ArgumentError
+                # Move operation was attempted at same time as another consumer;
+                # skip since the other process succeeded where this one didn't
+              end
             end
           end
 
-          def make_in_progress(job, new_dir, in_progress_dir)
-            move_job(File.join(new_dir, job), File.join(in_progress_dir, job))
+          # Moves job file to inprogress directory and returns the full path
+          # if the job was successfully locked by this consumer
+          def make_in_progress(job, new_dir, in_progress_dir, queue_timeout)
+            basename, previous_attempts, * = file_info(job)
+
+            from = File.join(new_dir, job)
+            # Add a timestamp to mark when the job was started
+            to = File.join(in_progress_dir, "#{basename}.#{previous_attempts}.#{Time.now.to_i}.job")
+
+            # If the file is non-zero, this means it was successfully written to
+            # by a publisher and we can attempt to move it to "in progress".
+            # 
+            # There is a small window of time where the file can be zero, but
+            # the publisher hasn't finished writing to the file yet.
+            if !File.zero?(from)
+              File.open(from, "r") do |f|
+                # If the lock can't be obtained, that means it's been locked
+                # by another consumer or the publisher of the file) -- don't
+                # block and skip it
+                if f.flock(File::LOCK_EX | File::LOCK_NB)
+                  FileUtils.mv(from, to)
+                  to
+                end
+              end
+            elsif (Time.now - File.ctime(from)) >= queue_timeout
+              # The file is empty (zero bytes) and enough time has passed since
+              # the file was written that we can safely assume it will never
+              # get written to be the publisher.
+              # 
+              # The scenario where this happens is when the publisher created
+              # the file, but the process was killed before it had a chance to
+              # actually write the data.
+              File.delete(from)
+              nil
+            end
+          rescue Errno::ENOENT
+            # File no longer exists; skip it since it's been picked up by
+            # another consumer
           end
 
+          # Moves job file to new directory and returns the full path
           def make_new_again(job, new_dir, in_progress_dir)
             basename, previous_attempts = file_info(job)
-            move_job(File.join(in_progress_dir, job), File.join(new_dir, "#{basename}.#{previous_attempts + 1}.job"))
-          end
 
-          # Moves job file to inprogress directory and returns the full path
-          def move_job(from, to)
-            f = File.open(from, "r")
-            # wait on the lock a publisher in another process might have.
-            # Once we get the lock the file is ours to move to mark it in progress
-            f.flock(File::LOCK_EX)
-            begin
-              FileUtils.mv(f.path, to)
-            ensure
-              f.flock(File::LOCK_UN) # yes we can unlock it after its been moved, I checked
-            end
+            from = File.join(in_progress_dir, job)
+            to = File.join(new_dir, "#{basename}.#{previous_attempts + 1}.job")
+            FileUtils.mv(from, to)
+
             to
           end
 
-          def job_files(dir)
-            Dir.entries(dir).select{|e| ! e.start_with?(".")}
+          def each_file(path, limit = nil)
+            count = 0
+
+            Dir.foreach(path) do |file|
+              next if file.start_with?('.')
+
+              yield file
+
+              count += 1
+              break if limit && count >= limit
+            end
           end
 
           # Grabs the unique identifier for the job filename and the number of times
           # it's been attempted (also based on the filename)
           def file_info(job_file)
-            id, previous_attempts = File.basename(job_file, '.job').split('.')
-            [id, previous_attempts.to_i]
+            id, previous_attempts, timestamp, * = job_file.split('.')
+            [id, previous_attempts.to_i, timestamp.to_i]
           end
         end
 
+        # The minimum number of seconds to allow to pass between checks for expired
+        # jobs on the filesystem.
+        # 
+        # Since queue times are measured on the order of seconds, 1 second is the
+        # smallest duration.  It also prevents us from burning a lot of CPU looking
+        # at expired jobs when the consumer sleep interval is less than 1 second.
+        EXPIRATION_CHECK_INTERVAL = 1
+
         # The amount of time units of work can run before the queue considers
         # them timed out.  For filesystem queues, this is the global default.
         attr_reader :queue_timeout
@@ -77,38 +127,46 @@ module Chore
         def initialize(queue_name, opts={})
           super(queue_name, opts)
 
-          # Even though putting these Mutexes in this hash is, by itself, not particularly threadsafe
-          # as long as some Mutex ends up in the queue after all consumers are created we're good
-          # as they are pulled from the queue and synchronized for file operations below
-          FILE_QUEUE_MUTEXES[@queue_name] ||= Mutex.new
-
           @in_progress_dir = self.class.in_progress_dir(queue_name)
           @new_dir = self.class.new_dir(queue_name)
-          @queue_timeout = Chore.config.default_queue_timeout
+          @queue_timeout = self.class.queue_timeout(queue_name)
         end
 
-        def consume(&handler)
+        def consume
           Chore.logger.info "Starting consuming file system queue #{@queue_name} in #{self.class.queue_dir(queue_name)}"
           while running?
             begin
-              #TODO move expired job files to new directory?
-              handle_jobs(&handler)
+              # Move expired job files to new directory (so long as enough time has
+              # passed since we last did this check)
+              if !@last_cleaned_at || (Time.now - @last_cleaned_at).to_i >= EXPIRATION_CHECK_INTERVAL
+                self.class.cleanup(Time.now.to_i - @queue_timeout, @new_dir, @in_progress_dir)
+                @last_cleaned_at = Time.now
+              end
+
+              found_files = false
+              handle_jobs do |*args|
+                found_files = true
+                yield(*args)
+              end
             rescue => e
               Chore.logger.error { "#{self.class}#consume: #{e} #{e.backtrace * "\n"}" }
             ensure
-              sleep 5
+              sleep(Chore.config.consumer_sleep_interval) unless found_files
             end
           end
         end
 
+        # Rejects the given message from the filesystem by +id+. Currently a noop
         def reject(id)
-          Chore.logger.debug "Rejecting: #{id}"
-          make_new_again(id)
+
         end
 
         def complete(id)
           Chore.logger.debug "Completing (deleting): #{id}"
-          FileUtils.rm(File.join(@in_progress_dir, id))
+          File.delete(File.join(@in_progress_dir, id))
+        rescue Errno::ENOENT
+          # The job took too long to complete, was deemed expired, and moved
+          # back into "new".  Ignore.
         end
 
         private
@@ -116,27 +174,26 @@ module Chore
         # finds all new job files, moves them to in progress and starts the job
         # Returns a list of the job files processed
         def handle_jobs(&block)
-          # all consumers on a single queue share a lock on handling files.
-          # Each consumer comes along, processes all present files and release the lock.
-          # This isn't particularly useful but is here to allow the configuration of
-          # ThreadedConsumerStrategy with mutiple threads on a queue safely although you
-          # probably wouldn't want to do that.
-          FILE_QUEUE_MUTEXES[@queue_name].synchronize do
-            self.class.job_files(@new_dir).each do |job_file|
-              Chore.logger.debug "Found a new job #{job_file}"
-              
-              job_json = File.read(make_in_progress(job_file))
-              basename, previous_attempts = self.class.file_info(job_file)
-
-              # job_file is just the name which is the job id
-              block.call(job_file, queue_name, queue_timeout, job_json, previous_attempts)
-              Chore.run_hooks_for(:on_fetch, job_file, job_json)
-            end
+          self.class.each_file(@new_dir, Chore.config.queue_polling_size) do |job_file|
+            Chore.logger.debug "Found a new job #{job_file}"
+
+            in_progress_path = make_in_progress(job_file)
+            next unless in_progress_path
+
+            # The job filename may have changed, so update it to reflect the in progress path
+            job_file = File.basename(in_progress_path)
+
+            job_json = File.read(in_progress_path)
+            basename, previous_attempts, * = self.class.file_info(job_file)
+
+            # job_file is just the name which is the job id
+            block.call(job_file, queue_name, queue_timeout, job_json, previous_attempts)
+            Chore.run_hooks_for(:on_fetch, job_file, job_json)
           end
         end
 
         def make_in_progress(job)
-          self.class.make_in_progress(job, @new_dir, @in_progress_dir)
+          self.class.make_in_progress(job, @new_dir, @in_progress_dir, @queue_timeout)
         end
 
         def make_new_again(job)