chore-core 1.8.2 → 4.0.0

data/chore-core.gemspec

@@ -37,10 +37,10 @@ Gem::Specification.new do |s|
   s.summary = "Job processing... for the future!"
 
   s.add_runtime_dependency(%q<json>, [">= 0"])
-  s.add_runtime_dependency(%q<aws-sdk-v1>, ["~> 1.56", ">= 1.56.0"])
+  s.add_runtime_dependency(%q<aws-sdk-sqs>, ["~> 1"])
   s.add_runtime_dependency(%q<thread>, ["~> 0.1.3"])
-  s.add_development_dependency(%q<rspec>, ["~> 3.3.0"])
+  s.add_runtime_dependency('get_process_mem', ["~> 0.2.0"])
+  s.add_development_dependency(%q<rspec>, ["~> 3.3"])
   s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
   s.add_development_dependency(%q<bundler>, [">= 0"])
 end
-

data/lib/chore.rb

@@ -34,13 +34,18 @@ module Chore #:nodoc:
     :fetcher               => Fetcher,
     :consumer_strategy     => Strategy::ThreadedConsumerStrategy,
     :batch_size            => 50,
+    :batch_timeout         => 20,
     :log_level             => Logger::WARN,
     :log_path              => STDOUT,
     :default_queue_timeout => (12 * 60 * 60), # 12 hours
     :shutdown_timeout      => (2 * 60),
     :max_attempts          => 1.0 / 0.0, # Infinity
     :dupe_on_cache_failure => false,
-    :payload_handler => Chore::Job
+    :queue_polling_size    => 10,
+    :payload_handler       => Chore::Job,
+    :master_procline       => "chore-master-#{Chore::VERSION}",
+    :worker_procline       => "chore-worker-#{Chore::VERSION}",
+    :consumer_sleep_interval => 1
   }
 
   class << self
@@ -58,6 +63,24 @@ module Chore #:nodoc:
     end
   end
 
+  def self.log_level_to_sym
+    return self.config[:log_level] if self.config[:log_level].is_a?(Symbol)
+    case self.config[:log_level]
+    when 0
+      :debug
+    when 1
+      :info
+    when 2
+      :warn
+    when 3
+      :error
+    when 4
+      :fatal
+    else
+      :unknown
+    end
+  end
+
   # Reopens any open files.  This will match any logfile that was opened by Chore,
   # Rails, or any other library.
   def self.reopen_logs
@@ -110,9 +133,9 @@ module Chore #:nodoc:
   #   add_hook(:before_fork) {|worker| puts 1 }
   #   add_hook(:before_fork) {|worker| puts 2 }
   #   add_hook(:before_fork) {|worker| puts 3 }
-  #   
+  #
   #   run_hooks_for(:before_fork, worker)
-  #   
+  #
   #   # ...will produce the following output
   #   => 1
   #   => 2
@@ -129,9 +152,9 @@ module Chore #:nodoc:
   #   add_hook(:around_fork) {|worker, &block| puts 'before 1'; block.call; puts 'after 1'}
   #   add_hook(:around_fork) {|worker, &block| puts 'before 2'; block.call; puts 'after 2'}
   #   add_hook(:around_fork) {|worker, &block| puts 'before 3'; block.call; puts 'after 3'}
-  #   
+  #
   #   run_hooks_for(:around_fork, worker) { puts 'block' }
-  #   
+  #
   #   # ...will produce the following output
   #   => before 1
   #   => before 2
@@ -186,6 +209,7 @@ module Chore #:nodoc:
   #   Chore.configure do |c|
   #     c.consumer = Chore::Queues::SQS::Consumer
   #     c.batch_size = 50
+  #     c.batch_timeout = 20
   #   end
   def self.configure(opts={})
     @config = (@config ? @config.merge_hash(opts) : Chore::Configuration.new(DEFAULT_OPTIONS.merge(opts)))
@@ -212,6 +236,8 @@ module Chore #:nodoc:
   end
 
   # List of queue_names as configured via Chore::Job including their prefix, if set.
+  #
+  # @return [Array<String>]
   def self.prefixed_queue_names
     Chore::Job.job_classes.collect {|klass| c = constantize(klass); c.prefixed_queue_name}
   end

data/lib/chore/cli.rb

@@ -89,10 +89,11 @@ module Chore #:nodoc:
       detect_queues
       Chore.configure(options)
       Chore.configuring = false
+      validate_strategy!
     end
 
-
     private
+
     def setup_options #:nodoc:
       register_option "queues", "-q", "--queues QUEUE1,QUEUE2", "Names of queues to process (default: all known)" do |arg|
         # This will remove duplicates. We ultimately force this to be a Set further below
@@ -134,7 +135,7 @@ module Chore #:nodoc:
         options[:consumer_strategy] = constantize(arg)
       end
 
-      register_option 'consumer_sleep_interval', '--consumer-sleep-interval INTERVAL', Float, 'Length of time in seconds to sleep when the consumer does not find any messages. Defaults vary depending on consumer implementation'
+      register_option 'consumer_sleep_interval', '--consumer-sleep-interval INTERVAL', Float, 'Length of time in seconds to sleep when the consumer does not find any messages (default: 1)'
 
       register_option 'payload_handler', '--payload_handler CLASS_NAME', 'Name of a class to use as the payload handler (default: Chore::Job)' do |arg|
         options[:payload_handler] = constantize(arg)
@@ -144,6 +145,7 @@ module Chore #:nodoc:
 
       register_option 'dupe_on_cache_failure', '--dupe-on-cache-failure BOOLEAN', 'Determines the deduping behavior when a cache connection error occurs. When set to false, the message is assumed not to be a duplicate. (default: false)'
 
+      register_option 'queue_polling_size', '--queue_polling_size NUM', Integer, 'Amount of messages to grab on each request (default: 10)'
     end
 
     def parse_opts(argv, ignore_errors = false) #:nodoc:
@@ -254,7 +256,6 @@ module Chore #:nodoc:
     end
 
     def validate! #:nodoc:
-
       missing_option!("--require [PATH|DIR]") unless options[:require]
 
       if !File.exist?(options[:require]) ||
@@ -266,8 +267,25 @@ module Chore #:nodoc:
         puts @parser
         exit(1)
       end
+    end
 
+    def validate_strategy!
+      consumer_strategy = Chore.config.consumer_strategy.to_s
+      worker_strategy = Chore.config.worker_strategy.to_s
+
+      throttled_consumer = 'Chore::Strategy::ThrottledConsumerStrategy'
+      preforked_worker = 'Chore::Strategy::PreForkedWorkerStrategy'
+
+      if consumer_strategy == throttled_consumer || worker_strategy == preforked_worker
+        unless consumer_strategy == throttled_consumer && worker_strategy == preforked_worker
+          puts "=================================================================="
+          puts "  PreForkedWorkerStrategy may only be paired with   "
+          puts "  ThrottledConsumerStrategy or vice versa  "
+          puts "  Please check your configurations "
+          puts "=================================================================="
+          exit(1)
+        end
+      end
     end
   end
 end
-

data/lib/chore/configuration.rb

@@ -1,6 +1,6 @@
 module Chore
   # Wrapper around an OpenStruct to define configuration data
-  # (TODO): Add required opts, and validate that they're set
+  # TODO: Add required opts, and validate that they're set
   class Configuration < OpenStruct
     # Helper method to make merging Hashes into OpenStructs easier
     def merge_hash(hsh={})

data/lib/chore/consumer.rb

@@ -1,16 +1,17 @@
 module Chore
   # Raised when Chore is booting up, but encounters a set of configuration that is impossible to boot from. Typically
-  # you'll find additional information around the cause of the exception by examining the logfiles
+  # you'll find additional information around the cause of the exception by examining the logfiles.
+  # You can raise this exception if your queue is in a terrible state and must shut down.
   class TerribleMistake < Exception
-    # You can raise this exception if your queue is in a terrible state and must shut down
   end
 
-  # Base class for a Chore Consumer. Provides the basic interface to adhere to for building custom
-  # Chore Consumers.
+  # Base class for a Chore Consumer. Provides the interface that a Chore::Consumer implementation should adhere to.
   class Consumer
 
     attr_accessor :queue_name
 
+    # @param [String] queue_name Name of queue to be consumed from
+    # @param [Hash] opts
     def initialize(queue_name, opts={})
       @queue_name = queue_name
       @running = true
@@ -21,26 +22,28 @@ module Chore
     def self.reset_connection!
     end
 
-    # Cleans up any resources that were left behind from prior instances of the
-    # chore process.  By default, this is a no-op.
-    def self.cleanup(queue)
-    end
-
     # Consume takes a block with an arity of two. The two params are
     # |message_id,message_body| where message_id is any object that the
     # consumer will need to be able to act on a message later (reject, complete, etc)
-    def consume(&block)
+    #
+    # @param [Block] &handler Message handler, used by the calling context (worker) to create & assigns a UnitOfWork
+    def consume(&handler)
       raise NotImplementedError
     end
 
     # Reject should put a message back on a queue to be processed again later. It takes
     # a message_id as returned via consume.
+    #
+    # @param [String] message_id Unique ID of the message
     def reject(message_id)
       raise NotImplementedError
     end
 
-    # Complete should mark a message as finished. It takes a message_id as returned via consume
-    def complete(message_id)
+    # Complete should mark a message as finished.
+    #
+    # @param [String] message_id Unique ID of the message
+    # @param [Hash] receipt_handle Unique ID of the consuming transaction in non-filesystem implementations
+    def complete(message_id, receipt_handle)
       raise NotImplementedError
     end
 
@@ -50,8 +53,47 @@ module Chore
     end
 
     # Returns true if the Consumer is currently running
+    #
+    # @return [TrueClass, FalseClass]
     def running?
       @running
     end
+
+    # Returns up to n work
+    #
+    # @param n
+    def provide_work(n)
+      raise NotImplementedError
+    end
+
+    # Determine whether or not we have already seen this message
+    #
+    # @param [String] dedupe_key
+    # @param [Class] klass
+    # @param [Integer] queue_timeout
+    #
+    # @return [TrueClass, FalseClass]
+    def duplicate_message?(dedupe_key, klass, queue_timeout)
+      dupe_detector.found_duplicate?(:id=>dedupe_key, :queue=>klass.to_s, :visibility_timeout=>queue_timeout)
+    end
+
+    # Instance of duplicate detection implementation class
+    #
+    # @return [DuplicateDetector]
+    def dupe_detector
+      @dupes ||= DuplicateDetector.new({:servers => Chore.config.dedupe_servers,
+                                        :dupe_on_cache_failure => false})
+    end
+
+    private
+
+    # Gets messages from queue implementation and invokes the provided block over each one. Afterwards, the :on_fetch
+    # hook will be invoked per message. This block call provides data necessary for the worker (calling context) to
+    # populate a UnitOfWork struct.
+    #
+    # @param [Block] &handler Message handler, passed along by #consume
+    def handle_messages(&handler)
+      raise NotImplementedError
+    end
   end
 end

data/lib/chore/fetcher.rb

@@ -11,21 +11,16 @@ module Chore
     # Starts the fetcher with the configured Consumer Strategy. This will begin consuming messages from your queue
     def start
       Chore.logger.info "Fetcher starting up"
-
-      # Clean up configured queues in case there are any resources left behind
-      Chore.config.queues.each do |queue|
-        Chore.config.consumer.cleanup(queue)
-      end
-
       @strategy.fetch
     end
 
     # Stops the fetcher, preventing any further messages from being pulled from the queue
     def stop!
       unless @stopping
-        Chore.logger.info "Fetcher shutting down"
+        Chore.logger.info "Fetcher shutting down started"
         @stopping = true
         @strategy.stop!
+        Chore.logger.info "Fetcher shutting down completed"
       end
     end
 
@@ -33,5 +28,15 @@ module Chore
     def stopping?
       @stopping
     end
+
+    # returns upto n work units
+    def provide_work(n)
+      @strategy.provide_work(n)
+    end
+
+    # gives work back to the consumer in case it couldn't be assigned
+    def return_work(work_units)
+      @strategy.return_work(work_units)
+    end
   end
 end

data/lib/chore/hooks.rb

@@ -15,7 +15,8 @@ module Chore
 
   private
     def hooks_for(event)
-      candidate_methods.grep(/^#{event}/).sort
+      @_chore_hooks ||= {}
+      @_chore_hooks[event] ||= candidate_methods.grep(/^#{event}/).sort
     end
 
     # NOTE: Any hook methods defined after this is first referenced (i.e.,

data/lib/chore/job.rb

@@ -60,6 +60,12 @@ module Chore
             raise ArgumentError, "#{self.to_s}: backoff must accept a single argument"
           end
         end
+
+        if @chore_options.key?(:dedupe_lambda)
+          if !@chore_options[:dedupe_lambda].is_a?(Proc)
+            raise ArgumentError, "#{self.to_s}: dedupe_lambda must be a lambda or Proc"
+          end
+        end
       end
 
       # This is a method so it can be overriden to create additional required
@@ -99,6 +105,8 @@ module Chore
       end
 
       # The name of the configured queue, combined with an optional prefix
+      #
+      # @return [String]
       def prefixed_queue_name
         "#{Chore.config.queue_prefix}#{self.options[:name]}"
       end
@@ -108,6 +116,17 @@ module Chore
       def has_backoff?
         self.options.key?(:backoff)
       end
+
+      def has_dedupe_lambda?
+        self.options.key?(:dedupe_lambda)
+      end
+
+      def dedupe_key(*args)
+        return unless has_dedupe_lambda?
+
+        # run the proc to get the key
+        self.options[:dedupe_lambda].call(*args).to_s
+      end
     end #ClassMethods
 
     # This is handy to override in an included job to be able to do job setup that requires

data/lib/chore/manager.rb

@@ -5,19 +5,21 @@ require 'chore/fetcher'
 module Chore
   # Manages the interactions between fetching messages (Consumer Strategy), and working over them (Worker Strategy)
   class Manager
+    include Util
 
     def initialize()
       Chore.logger.info "Booting Chore #{Chore::VERSION}"
       Chore.logger.debug { Chore.config.inspect }
+      procline("#{Chore.config.master_procline}:Started:#{Time.now}")
       @started_at = nil
       @worker_strategy = Chore.config.worker_strategy.new(self)
       @fetcher = Chore.config.fetcher.new(self)
-      @processed = 0
       @stopping = false
     end
 
     # Start the Manager. This calls both the #start method of the configured Worker Strategy, as well as Fetcher#start.
     def start
+      Chore.run_hooks_for(:before_start)
       @started_at = Time.now
       @worker_strategy.start
       @fetcher.start
@@ -26,11 +28,12 @@ module Chore
     # Shut down the Manager, the Worker Strategy, and the Fetcher. This calls the +:before_shutdown+ hook.
     def shutdown!
       unless @stopping
-        Chore.logger.info "Manager shutting down"
+        Chore.logger.info "Manager shutting down started"
         @stopping = true
         Chore.run_hooks_for(:before_shutdown)
         @fetcher.stop!
         @worker_strategy.stop!
+        Chore.logger.info "Manager shutting down completed"
       end
     end
 
@@ -41,7 +44,20 @@ module Chore
     # than they can be consumed.
     def assign(work)
       Chore.logger.debug { "Manager#assign: No. of UnitsOfWork: #{work.length})" }
+      work.each do | item |
+        Chore.run_hooks_for(:before_send_to_worker, item)
+      end
       @worker_strategy.assign(work) unless @stopping
     end
+
+    # returns up to n from the throttled consumer queue
+    def fetch_work(n)
+      @fetcher.provide_work(n)
+    end
+
+    # gives work back to the fetcher in case it couldn't be assigned
+    def return_work(work_units)
+      @fetcher.return_work(work_units)
+    end
   end
 end

data/lib/chore/publisher.rb

@@ -1,26 +1,42 @@
 module Chore
-  # Base class for Chore Publishers. Provides the bare interface one needs to adhere to when writing custom publishers
+  # Base class for a Chore Publisher. Provides the interface that a Chore::Publisher implementation should adhere to.
   class Publisher
     DEFAULT_OPTIONS = { :encoder => Encoder::JsonEncoder }
 
     attr_accessor :options
 
+    # @param [Hash] opts
     def initialize(opts={})
       self.options = DEFAULT_OPTIONS.merge(opts)
     end
 
     # Publishes the provided +job+ to the queue identified by the +queue_name+. Not designed to be used directly, this
     # method ferries to the publish method on an instance of your configured Publisher.
+    #
+    # @param [String] queue_name Name of queue to be consumed from
+    # @param [Hash] job Job instance definition, will be encoded to JSON
     def self.publish(queue_name,job)
       self.new.publish(queue_name,job)
     end
 
-    # Raises a NotImplementedError. This method should be overridden in your descendent, custom publisher class
+    # Publishes a message to queue
+    #
+    # @param [String] queue_name Name of the SQS queue
+    # @param [Hash] job Job instance definition, will be encoded to JSON
     def publish(queue_name,job)
       raise NotImplementedError
     end
+
+    # Sets a flag that instructs the publisher to reset the connection the next time it's used.
+    # Should be overriden in publishers (but is not required)
+    def self.reset_connection!
+    end
+
   protected
 
+    # Encodes the job class to format provided by endoder implementation
+    #
+    # @param [Any] job
     def encode_job(job)
       options[:encoder].encode(job)
     end