chore-core 1.5.2

data/lib/chore/strategies/worker/forked_worker_strategy.rb

@@ -0,0 +1,221 @@
+require 'chore/signal'
+
+module Chore
+  module Strategy
+    class ForkedWorkerStrategy #:nodoc:
+      attr_accessor :workers
+
+      def initialize(manager)
+        @manager = manager
+        @stopped = false
+        @workers = {}
+        @queue = Queue.new
+        Chore.config.num_workers.times { @queue << :worker }
+
+        trap_master_signals
+        monitor_workers
+
+        Chore.run_hooks_for(:before_first_fork)
+      end
+
+      # Start up the worker strategy. In this particular case, what we're doing
+      # is starting up a WorkerListener, so we can talk to the children.
+      def start
+        Chore.logger.debug "Starting up worker strategy: #{self.class.name}"
+      end
+
+      # Stop the workers. The particulars of the implementation here are that we
+      # send a QUIT signal to each child, wait one minute for it to finish the last job
+      # it was working on. If it times out, then we send KILL. In an ideal world this means
+      # that <tt>stop!</tt> is non-destructive in that it allow each worker to complete it's
+      # current job before dying.
+      def stop!
+        return if @stopped
+
+        @stopped = true
+        Chore.logger.info { "Manager #{Process.pid} stopping" }
+
+        # Instead of using Process.waitall (which is a blocking operation that can
+        # cause the master process to hang), use a Unicorn style non-blocking
+        # shutdown process.
+        limit = Time.now + Chore.config.shutdown_timeout
+        until workers.empty? || Time.now > limit
+          signal_children("QUIT")
+          sleep(0.1)
+          reap_terminated_workers!
+        end
+
+        if !workers.empty?
+          Chore.logger.error "Timed out waiting for children to terminate. Terminating with prejudice."
+          signal_children("KILL")
+        end
+      end
+
+      # Take a UnitOfWork (or an Array of UnitOfWork) and assign it to a Worker. We only
+      # assign work if there are <tt>workers_available?</tt>.
+      def assign(work)
+        return unless acquire_worker
+
+        begin
+          w = Worker.new(work)
+          Chore.run_hooks_for(:before_fork,w)
+          pid = nil
+          Chore.run_hooks_for(:around_fork,w) do
+            pid = fork do
+              after_fork(w)
+              Chore.run_hooks_for(:within_fork,w) do
+                Chore.run_hooks_for(:after_fork,w)
+                begin
+                  Chore.logger.info("Started worker:#{Time.now}")
+                  w.start
+                  Chore.logger.info("Finished worker:#{Time.now}")
+                ensure
+                  Chore.run_hooks_for(:before_fork_shutdown)
+                  exit!(true)
+                end
+              end #within_fork
+            end #around_fork
+          end
+
+          Chore.logger.debug { "Forked worker #{pid}"}
+          workers[pid] = w
+        rescue => ex
+          Chore.logger.error { "Failed to fork worker: #{ex.message} #{ex.backtrace * "\n"}"}
+          release_worker
+        end
+      end
+
+      private
+
+      def trap_master_signals
+        Signal.trap('CHLD') { reap_terminated_workers! }
+      end
+
+      def trap_child_signals(worker)
+        # Register a new QUIT handler to make the current worker
+        # finish this job, and not complete another one.
+        Signal.trap("INT") { worker.stop! }
+        Signal.trap("QUIT") { worker.stop! }
+        #By design, we do nothing in children on USR1, so we are not re-defining this like we do INT and QUIT
+      end
+
+      def clear_child_signals
+        # Remove handlers from the parent process
+        Signal.reset
+      end
+
+      # Attempts to essentially acquire a lock on a worker.  If no workers are
+      # available, then this will block until one is.
+      def acquire_worker
+        result = @queue.pop
+
+        if @stopped
+          # Strategy has stopped since the worker was acquired.  If workers are
+          # allowed to run even though the strategy is stopped, this could result
+          # in forks occuring while the CLI is calling +Kernel#exit+ -- which can
+          # cause chore to hang.
+          release_worker
+          nil
+        else
+          result
+        end
+      end
+
+      # Releases the lock on a worker so that another thread can pick it up.
+      def release_worker
+        @queue << :worker
+      end
+
+      # Only call this in the forked child. It resets some things that need fixing up
+      # in the child.
+      def after_fork(worker)
+        # Immediately swap out the process name so that it doesn't look like
+        # the master process
+        procline("Started:#{Time.now}")
+
+        clear_child_signals
+        trap_child_signals(worker)
+
+        # We need to reset the logger after fork. This fixes a longstanding bug
+        # where workers would hang around and never die
+        Chore.logger = nil
+
+        # When we fork, the consumer's / publisher's need their connections reset. The specifics of this
+        # are queue dependent, and may result in a noop.
+        Chore.config.consumer.reset_connection!
+        Chore.config.publisher.reset_connection! if Chore.config.publisher #It is possible for this to be nil due to configuration woes with chore
+      end
+
+      # Reaps any in-flight workers that have completed.  This only relies on
+      # known process ids instead of discovering all child processes from the
+      # OS.  By doing this, we avoid running into a tight loop reaping
+      # short-lived forks.
+      def reap_terminated_workers!
+        pids.each do |pid|
+          reaped = false
+          begin
+            reaped = Process.wait(pid, Process::WNOHANG)
+          rescue Errno::ECHILD => ex
+            # Child process has already terminated
+            reaped = true
+          end
+
+          # Clean up / release worker
+          if reaped && workers.delete(pid)
+            release_worker
+            Chore.logger.debug { "Removed finished worker #{pid}"}
+          end
+        end
+      end
+
+      # Ensures that workers don't live past their expiration date.  When this
+      # is detect, the workers are kill -9'd.
+      def monitor_workers
+        Thread.new do
+          while !@stopped
+            pids.each do |pid|
+              worker = workers[pid]
+              if worker && worker.expired?
+                messages = worker.work.map(&:message)
+                Chore.logger.error { "Failed to run jobs -- #{messages} -- by #{worker.expires_at}, started at #{worker.started_at}"}
+                Chore.run_hooks_for(:on_failure, {:body => 'Worker fork timed out.', :messages => messages}, TimeoutError.new)
+
+                signal_children('KILL', [pid])
+              end
+            end
+
+            sleep 1
+          end
+        end
+      end
+
+      # Take a snapshot in time of what workers are in flight.  Iterating over
+      # the workers collection itself can lead to a hard loop if new items are
+      # constantly added.
+      def pids
+        workers.keys
+      end
+
+      # Wrapper around fork for specs.
+      def fork(&block)
+        Kernel.fork(&block)
+      end
+
+      def procline(str)
+        Chore.logger.info str
+        $0 = "chore-#{Chore::VERSION}:#{str}"
+      end
+
+      def signal_children(sig, pids_to_signal = pids)
+        pids_to_signal.each do |pid|
+          begin
+            Chore.logger.info { "Sending #{sig} to: #{pid}" }
+            Process.kill(sig, pid)
+          rescue Errno::ESRCH
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/chore/strategies/worker/single_worker_strategy.rb

@@ -0,0 +1,39 @@
+module Chore
+  module Strategy
+    
+    # Worker strategy for performing batches of work in a linear fashion. Ideally used for running
+    # Chore jobs locally in a development environment where performance or throughput may not matter.
+    class SingleWorkerStrategy
+
+      attr_reader :worker
+
+      def initialize(manager)
+        @manager = manager
+        @worker = nil
+      end
+
+      # Starts the <tt>SingleWorkerStrategy</tt>. Currently a noop
+      def start;end
+
+      # Stops the <tt>SingleWorkerStrategy</tt> if there is a worker to stop
+      def stop!
+        worker.stop! if worker
+      end
+
+      # Assigns work if there isn't already a worker in progress. Otherwise, is a noop
+      def assign(work)
+        if workers_available?
+          @worker = Worker.new(work)
+          @worker.start
+          @worker = nil
+          true
+        end
+      end
+
+      # Returns true if there is currently no worker
+      def workers_available?
+        @worker.nil?
+      end
+    end
+  end
+end

data/lib/chore/tasks/queues.task

@@ -0,0 +1,11 @@
+namespace :chore do
+  desc "Create all defined queues"
+  task :create do
+    Chore::Queues::SQS.create_queues!
+  end
+
+  desc "Remove all defined queues"
+  task :remove do
+    Chore::Queues::SQS.delete_queues!
+  end
+end

data/lib/chore/unit_of_work.rb

@@ -0,0 +1,17 @@
+module Chore
+  # Simple class to hold job processing information.
+  # Has six attributes:
+  # * +:id+ The queue implementation specific identifier for this message.
+  # * +:queue_name+ The name of the queue the job came from
+  # * +:queue_timeout+ The time (in seconds) before the job will get re-enqueued if not processed
+  # * +:message+ The actual data of the message.
+  # * +:previous_attempts+ The number of times the work has been attempted previously.
+  # * +:consumer+ The consumer instance used to fetch this message. Most queue implementations won't need access to this, but some (RabbitMQ) will. So we
+  # make sure to pass it along with each message. This instance will be used by the Worker for things like <tt>complete</tt> and </tt>reject</tt>.
+  class UnitOfWork < Struct.new(:id,:queue_name,:queue_timeout,:message,:previous_attempts,:consumer)
+    # The current attempt number for the worker processing this message.
+    def current_attempt
+      previous_attempts + 1
+    end
+  end
+end

data/lib/chore/util.rb

@@ -0,0 +1,18 @@
+module Chore
+
+  # Collection of utilities and helpers used by Chore internally
+  module Util
+    
+    # To avoid bringing in all of active_support, we implemented constantize here
+    def constantize(camel_cased_word)
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name|
+        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+      end
+      constant
+    end
+  end
+end

data/lib/chore/version.rb

@@ -0,0 +1,9 @@
+module Chore
+  module Version #:nodoc:
+    MAJOR = 1
+    MINOR = 5
+    PATCH = 2
+
+    STRING = [ MAJOR, MINOR, PATCH ].join('.')
+  end
+end

data/lib/chore/worker.rb

@@ -0,0 +1,117 @@
+require 'chore/util'
+require 'chore/json_encoder'
+
+module Chore
+  class TimeoutError < StandardError
+  end
+
+  # Worker is one of the core classes in Chore. It's responsible for most of the logic
+  # relating to actually processing a job. A given worker will take an amount of +work+
+  # and then process it all until either the worker is told to stop, or the work is
+  # completed. Worker is completely agnostic to the WorkerStrategy that it was called from.
+  class Worker
+    include Util
+
+    DEFAULT_OPTIONS = { :encoder => JsonEncoder }
+    attr_accessor :options
+    attr_reader   :work
+    attr_reader   :started_at
+
+    def self.start(work) #:nodoc:
+      self.new(work).start
+    end
+
+    # Create a Worker. Give it an array of work (or single item), and +opts+.
+    # Currently, the only option supported by Worker is +:encoder+ which should match
+    # the +:encoder+ used by the Publisher who created the message.
+    def initialize(work=[],opts={})
+      @stopping = false
+      @started_at = Time.now
+      @work = work
+      @work = [work] unless work.kind_of?(Array)
+      self.options = DEFAULT_OPTIONS.merge(opts)
+    end
+
+    # Whether this worker has existed for longer than it's allowed to
+    def expired?
+      Time.now > expires_at
+    end
+
+    # The time at which this worker expires
+    def expires_at
+      total_timeout = @work.inject(0) {|sum, item| sum += item.queue_timeout}
+      @started_at + total_timeout
+    end
+
+    # The workhorse. Do the work, all of it. This will block for an entirely unspecified amount
+    # of time based on the work to be performed. This will:
+    # * Decode each message.
+    # * Re-ify the messages into actual Job classes.
+    # * Call Job#perform on each job.
+    # * If successful it will call Consumer#complete (using the consumer in the UnitOfWork).
+    # * If unsuccessful it will call the appropriate Hooks based on the type of failure.
+    # * If unsuccessful *and* the maximum number of attempts for the job has been surpassed, it will call
+    #   the permanent failure hooks and Consumer#complete.
+    # * Log the results via the Chore.logger
+    def start
+      @work.each do |item|
+        return if @stopping
+        Chore.logger.debug { "Doing: #{item.queue_name} with #{item.message}" }
+        begin
+          start_item(item)
+        rescue => e
+          Chore.logger.error { "Failed to run job for #{item.message} with error: #{e.message} #{e.backtrace * "\n"}" }
+          if item.current_attempt >= Chore.config.max_attempts
+            Chore.run_hooks_for(:on_permanent_failure,item.queue_name,item.message,e)
+            item.consumer.complete(item.id)
+          else
+            Chore.run_hooks_for(:on_failure,item.message,e)
+          end
+        end
+      end
+    end
+
+    # Tell the worker to stop after it completes the current job.
+    def stop!
+      @stopping = true
+    end
+
+  protected
+    def payload_class(message)
+      constantize(message['class'])
+    end
+
+  private
+    def start_item(item)
+      message = decode_job(item.message)
+      klass = payload_class(message)
+      return unless klass.run_hooks_for(:before_perform,message)
+
+      begin
+        perform_job(klass,message)
+        item.consumer.complete(item.id)
+        klass.run_hooks_for(:after_perform,message)
+      rescue Job::RejectMessageException
+        item.consumer.reject(item.id)
+        Chore.logger.error { "Failed to run job for #{item.message}  with error: Job raised a RejectMessageException" }
+        klass.run_hooks_for(:on_rejected, message)
+      rescue => e
+        Chore.logger.error { "Failed to run job #{item.message} with error: #{e.message} at #{e.backtrace * "\n"}" }
+        if item.current_attempt >= klass.options[:max_attempts]
+          klass.run_hooks_for(:on_permanent_failure,item.queue_name,message,e)
+          item.consumer.complete(item.id)
+        else
+          klass.run_hooks_for(:on_failure,message,e)
+        end
+      end
+    end
+
+    def perform_job(klass, message)
+      klass.perform(*message['args'])
+    end
+
+    def decode_job(data)
+      options[:encoder].decode(data)
+    end
+  end
+end

data/lib/chore-core.rb

@@ -0,0 +1 @@
+require 'chore'

data/lib/chore.rb

@@ -0,0 +1,218 @@
+require 'ostruct'
+require 'logger'
+# Require chore files
+require 'chore/version'
+
+require 'chore/unit_of_work'
+require 'chore/configuration'
+require 'chore/cli'
+require 'chore/consumer'
+require 'chore/job'
+require 'chore/json_encoder'
+require 'chore/manager'
+require 'chore/publisher'
+require 'chore/util'
+require 'chore/worker'
+require 'chore/publisher'
+
+# We have a number of things that can live here. I don't want to track
+['queues/**','strategies/**'].each do |p|
+  Dir[File.join(File.dirname(__FILE__),'chore',p,'*.rb')].each {|f| require f}
+end
+
+module Chore #:nodoc:
+  VERSION = Chore::Version::STRING #:nodoc:
+
+  # The default configuration options for Chore.
+  DEFAULT_OPTIONS = {
+    :require               => "./",
+    :num_workers           => 4,
+    :threads_per_queue     => 1,
+    :worker_strategy       => Strategy::ForkedWorkerStrategy,
+    :consumer              => Queues::SQS::Consumer,
+    :fetcher               => Fetcher,
+    :consumer_strategy     => Strategy::ThreadedConsumerStrategy,
+    :batch_size            => 50,
+    :log_level             => Logger::WARN,
+    :log_path              => STDOUT,
+    :default_queue_timeout => (12 * 60 * 60), # 12 hours
+    :shutdown_timeout      => (2 * 60),
+    :max_attempts          => 1.0 / 0.0, # Infinity
+    :dupe_on_cache_failure => false
+  }
+
+  class << self
+    attr_accessor :logger
+  end
+
+  # Access Chore's logger in a memoized fashion. Will create an instance of the logger if
+  # one doesn't already exist.
+  def self.logger
+    @logger ||= Logger.new(config.log_path).tap do |l|
+      l.level = config.log_level
+      l.formatter = lambda do |severity, datetime, progname, msg|
+         "[#{datetime} (#{Process.pid})] #{severity} : #{msg}\n"
+      end
+    end
+  end
+
+  # Reopens any open files.  This will match any logfile that was opened by Chore,
+  # Rails, or any other library.
+  def self.reopen_logs
+    # Find any open file in the process
+    files = []
+    ObjectSpace.each_object(File) {|file| files << file unless file.closed?}
+
+    files.each do |file|
+      begin
+        file.reopen(file.path, 'a+')
+        file.sync = true
+      rescue
+        # Can't reopen -- ignore / skip the file
+      end
+    end
+  end
+
+  # Add a global hook for +name+. Will run +&blk+ when the hook is executed.
+  # Global hooks are any hooks that don't have access to an instance of a job.
+  # See the docs on Hooks for a full list of global hooks.
+  #
+  # === Examples
+  #   Chore.add_hook_for(:after_fork) do
+  #     SomeDB.reset_connection!
+  #   end
+  def self.add_hook(name,&blk)
+    @@hooks ||= {}
+    (@@hooks[name.to_sym] ||= []) << blk
+  end
+
+  # A helper to get a list of all the hooks for a given +name+
+  def self.hooks_for(name)
+    @@hooks ||= {}
+    @@hooks[name.to_sym] || []
+  end
+
+  def self.clear_hooks! #:nodoc:
+    @@hooks = {}
+  end
+
+  # Run the global hooks associated with a particular +name+ passing all +args+ to the registered block.
+  #
+  # == Before / After hooks
+  #
+  # If this is invoked for before / after hooks (i.e. no block is passed), then the
+  # hooks will be invoked in the order in which they're defined.
+  #
+  # For example:
+  #
+  #   add_hook(:before_fork) {|worker| puts 1 }
+  #   add_hook(:before_fork) {|worker| puts 2 }
+  #   add_hook(:before_fork) {|worker| puts 3 }
+  #   
+  #   run_hooks_for(:before_fork, worker)
+  #   
+  #   # ...will produce the following output
+  #   => 1
+  #   => 2
+  #   => 3
+  #
+  # == Around hooks
+  #
+  # If this is invoked for around hooks (i.e. a block is passed), then the hooks
+  # will be invoked in the order in which they're defined, with the passed block
+  # being invoked last after the hooks yield.
+  #
+  # For example:
+  #
+  #   add_hook(:around_fork) {|worker, &block| puts 'before 1'; block.call; puts 'after 1'}
+  #   add_hook(:around_fork) {|worker, &block| puts 'before 2'; block.call; puts 'after 2'}
+  #   add_hook(:around_fork) {|worker, &block| puts 'before 3'; block.call; puts 'after 3'}
+  #   
+  #   run_hooks_for(:around_fork, worker) { puts 'block' }
+  #   
+  #   # ...will produce the following output
+  #   => before 1
+  #   => before 2
+  #   => before 3
+  #   => block
+  #   => after 3
+  #   => after 2
+  #   => after 1
+  #
+  # You can imagine the callback order to be U shaped where logic *prior* to yielding
+  # is called in the order it's defined and logic *after* yielding is called in
+  # reverse order.  At the bottom of the U is when the block passed into +run_hooks_for+
+  # gets invoked.
+  def self.run_hooks_for(name,*args,&block)
+    if block
+      run_around_hooks_for(name, args, &block)
+    else
+      hooks = self.hooks_for(name)
+      hooks.each {|h| h.call(*args, &block)} unless hooks.nil? || hooks.empty?
+    end
+  end
+
+  class << self
+    private
+    # Runs the global *around* hooks.  This is similar to +run_hooks_for+ except it
+    # passing a block into each hook.
+    def run_around_hooks_for(name, args, index = 0, &block)
+      hooks = self.hooks_for(name)
+
+      if hook = hooks[index]
+        hook.call(*args) do
+          # Once the hook yields, call the next one
+          run_around_hooks_for(name, args, index + 1, &block)
+        end
+      else
+        # There are no more hooks: call the black passed into +run_hooks_for+.
+        # After this is called, the hooks will then execute their logic after the
+        # yield in reverse order.
+        block.call
+      end
+    end
+  end
+
+  # Configure global chore options. Takes a hash for +opts+.
+  # This includes things like the current Worker Strategy (+:worker_strategy+), the default Consumer (+:consumer+), and the default Consumer Strategy(+:consumer_strategy).
+  # It's safe to call multiple times (will merge the new config, into the old)
+  # This is used by the command line parsing code to setup Chore.
+  # If a +block+ is given, <tt>configure</tt> will yield the config object, so you can set options directly.
+  # === Examples
+  #   Chore.configure({:worker_strategy => Chore::ForkedWorkerStrategy})
+  #
+  #   Chore.configure do |c|
+  #     c.consumer = Chore::Queues::SQS::Consumer
+  #     c.batch_size = 50
+  #   end
+  def self.configure(opts={})
+    @config = (@config ? @config.merge_hash(opts) : Chore::Configuration.new(DEFAULT_OPTIONS.merge(opts)))
+    yield @config if block_given?
+    @config
+  end
+
+  # Return the current Chore configuration as specified by <tt>configure</tt>. You can chain config options off of this to
+  # get access to current config data.
+  # === Examples
+  #   puts Chore.config.num_workers
+  def self.config
+    @config ||= self.configure
+  end
+
+  # Helper flag for rails/web app chore initializers to use so that chore does not re-load itself during requirement loading
+  def self.configuring?
+    @configuring ||= false
+  end
+
+  # Setter for chore to indicate that it's in the middle of configuring itself
+  def self.configuring=(value)
+    @configuring = value
+  end
+
+  # List of queue_names as configured via Chore::Job including their prefix, if set.
+  def self.prefixed_queue_names
+    Chore::Job.job_classes.collect {|klass| c = klass.constantize; c.prefixed_queue_name}
+  end
+end
+
+require 'chore/railtie' if defined?(Rails)