qless 0.9.3 → 0.10.0

data/lib/qless/worker/base.rb

@@ -0,0 +1,247 @@
+# Encoding: utf-8
+
+# Standard stuff
+require 'time'
+require 'logger'
+require 'thread'
+
+# Qless requires
+require 'qless'
+require 'qless/subscriber'
+
+module Qless
+  module Workers
+    JobLockLost = Class.new(StandardError)
+
+    class BaseWorker
+      attr_accessor :output, :reserver, :interval, :paused,
+                    :options, :sighup_handler
+
+      def initialize(reserver, options = {})
+        # Our job reserver and options
+        @reserver = reserver
+        @options = options
+
+        # SIGHUP handler
+        @sighup_handler = options.fetch(:sighup_handler) { lambda { } }
+
+        # Our logger
+        @log = options.fetch(:logger) do
+          @output = options.fetch(:output, $stdout)
+          Logger.new(output).tap do |logger|
+            logger.level = options.fetch(:log_level, Logger::WARN)
+            logger.formatter = options.fetch(:log_formatter) do
+              Proc.new { |severity, datetime, progname, msg| "#{datetime}: #{msg}\n" }
+            end
+          end
+        end
+
+        # The interval for checking for new jobs
+        @interval = options.fetch(:interval, 5.0)
+        @current_job_mutex = Mutex.new
+        @current_job = nil
+
+        # Default behavior when a lock is lost: stop after the current job.
+        on_current_job_lock_lost { shutdown(in_signal_handler=false) }
+      end
+
+      def log_level
+        @log.level
+      end
+
+      def safe_trap(signal_name, &cblock)
+        begin
+          trap(signal_name, cblock)
+        rescue ArgumentError
+          warn "Signal #{signal_name} not supported."
+        end
+      end
+
+      # The meaning of these signals is meant to closely mirror resque
+      #
+      # TERM: Shutdown immediately, stop processing jobs.
+      #  INT: Shutdown immediately, stop processing jobs.
+      # QUIT: Shutdown after the current job has finished processing.
+      # USR1: Kill the forked children immediately, continue processing jobs.
+      # USR2: Pause after this job
+      # CONT: Start processing jobs again after a USR2
+      #  HUP: Print current stack to log and continue
+      def register_signal_handlers
+        # Otherwise, we want to take the appropriate action
+        trap('TERM') { exit! }
+        trap('INT')  { exit! }
+        safe_trap('HUP') { sighup_handler.call }
+        safe_trap('QUIT') { shutdown(in_signal_handler=true) }
+        begin
+          trap('CONT') { unpause(in_signal_handler=true) }
+          trap('USR2') { pause(in_signal_handler=true) }
+        rescue ArgumentError
+          warn 'Signals USR2, and/or CONT not supported.'
+        end
+      end
+
+      # Return an enumerator to each of the jobs provided by the reserver
+      def jobs
+        return Enumerator.new do |enum|
+          loop do
+            begin
+              job = reserver.reserve
+            rescue Exception => error
+              # We want workers to durably stay up, so we don't want errors
+              # during job reserving (e.g. network timeouts, etc) to kill the
+              # worker.
+              log(:error,
+                "Error reserving job: #{error.class}: #{error.message}")
+            end
+
+            # If we ended up getting a job, yield it. Otherwise, we wait
+            if job.nil?
+              no_job_available
+            else
+              self.current_job = job
+              enum.yield(job)
+              self.current_job = nil
+            end
+
+            break if @shutdown
+          end
+        end
+      end
+
+      # Actually perform the job
+      def perform(job)
+        around_perform(job)
+      rescue JobLockLost
+        log(:warn, "Lost lock for job #{job.jid}")
+      rescue Exception => error
+        fail_job(job, error, caller)
+      else
+        try_complete(job)
+      end
+
+      # Allow middleware modules to be mixed in and override the
+      # definition of around_perform while providing a default
+      # implementation so our code can assume the method is present.
+      module SupportsMiddlewareModules
+        def around_perform(job)
+          job.perform
+        end
+
+        def after_fork
+        end
+      end
+
+      include SupportsMiddlewareModules
+
+      # Stop processing after this job
+      def shutdown(in_signal_handler=true)
+        @shutdown = true
+      end
+      alias stop! shutdown # so we can call `stop!` regardless of the worker type
+
+      # Pause the worker -- take no more new jobs
+      def pause(in_signal_handler=true)
+        @paused = true
+        procline("Paused -- #{reserver.description}", in_signal_handler=in_signal_handler)
+      end
+
+      # Continue taking new jobs
+      def unpause(in_signal_handler=true)
+        @paused = false
+      end
+
+      # Set the proceline. Not supported on all systems
+      def procline(value, in_signal_handler=true)
+        $0 = "Qless-#{Qless::VERSION}: #{value} at #{Time.now.iso8601}"
+        log(:debug, $PROGRAM_NAME) unless in_signal_handler
+      end
+
+      # Complete the job unless the worker has already put it into another state
+      # by completing / failing / etc. the job
+      def try_complete(job)
+        job.complete unless job.state_changed?
+      rescue Job::CantCompleteError => e
+        # There's not much we can do here. Complete fails in a few cases:
+        #   - The job is already failed (i.e. by another worker)
+        #   - The job is being worked on by another worker
+        #   - The job has been cancelled
+        #
+        # We don't want to (or are able to) fail the job with this error in
+        # any of these cases, so the best we can do is log the failure.
+        log(:error, "Failed to complete #{job.inspect}: #{e.message}")
+      end
+
+      def fail_job(job, error, worker_backtrace)
+        failure = Qless.failure_formatter.format(job, error, worker_backtrace)
+        log(:error, "Got #{failure.group} failure from #{job.inspect}\n#{failure.message}" )
+        job.fail(*failure)
+      rescue Job::CantFailError => e
+        # There's not much we can do here. Another worker may have cancelled it,
+        # or we might not own the job, etc. Logging is the best we can do.
+        log(:error, "Failed to fail #{job.inspect}: #{e.message}")
+      end
+
+      def deregister
+        uniq_clients.each do |client|
+          client.deregister_workers(client.worker_name)
+        end
+      end
+
+      def uniq_clients
+        @uniq_clients ||= reserver.queues.map(&:client).uniq
+      end
+
+      def on_current_job_lock_lost(&block)
+        @on_current_job_lock_lost = block
+      end
+
+      def listen_for_lost_lock
+        subscribers = uniq_clients.map do |client|
+          Subscriber.start(client, "ql:w:#{client.worker_name}", log: @log) do |_, message|
+            if message['event'] == 'lock_lost'
+              with_current_job do |job|
+                if job && message['jid'] == job.jid
+                  @on_current_job_lock_lost.call(job)
+                end
+              end
+            end
+          end
+        end
+
+        yield
+      ensure
+        subscribers.each(&:stop)
+      end
+
+    private
+
+      def log(type, msg)
+        @log.public_send(type, "#{Process.pid}: #{msg}")
+      end
+
+      def no_job_available
+        unless interval.zero?
+          procline("Waiting for #{reserver.description}", in_signal_handler=false)
+          log(:debug, "Sleeping for #{interval} seconds")
+          sleep interval
+        end
+      end
+
+      def with_current_job
+        @current_job_mutex.synchronize do
+          yield @current_job
+        end
+      end
+
+      def current_job=(job)
+        @current_job_mutex.synchronize do
+          @current_job = job
+        end
+      end
+
+      def reconnect_each_client
+        uniq_clients.each { |client| client.redis.client.reconnect }
+      end
+    end
+  end
+end

data/lib/qless/worker/forking.rb

@@ -0,0 +1,245 @@
+# Encoding: utf-8
+
+# Qless requires
+require 'qless'
+require 'qless/worker/base'
+require 'qless/worker/serial'
+require 'thread'
+
+module Qless
+  module Workers
+    class ForkingWorker < BaseWorker
+      # The child startup interval
+      attr_accessor :max_startup_interval
+
+      def initialize(reserver, options = {})
+        super(reserver, options)
+        # The keys are the child PIDs, the values are information about the
+        # worker, including its sandbox directory. This directory currently
+        # isn't used, but this sets up for having that eventually.
+        @sandboxes = {}
+
+        # Save our options for starting children
+        @options = options
+
+        # The max interval between when children start (reduces thundering herd)
+        @max_startup_interval = options[:max_startup_interval] || 10.0
+
+        # TODO: facter to figure out how many cores we have
+        @num_workers = options[:num_workers] || 1
+
+        # All the modules that have been applied to this worker
+        @modules = []
+
+        @sandbox_mutex = Mutex.new
+        # A queue of blocks that are postponed since we cannot get
+        # @sandbox_mutex in trap handler
+        @postponed_actions_queue = ::Queue.new
+      end
+
+      # Because we spawn a new worker, we need to apply all the modules that
+      # extend this one
+      def extend(mod)
+        @modules << mod
+        super(mod)
+      end
+
+      # Spawn a new child worker
+      def spawn
+        worker = SerialWorker.new(reserver, @options)
+        # We use 11 as the exit status so that it is something unique
+        # (rather than the common 1). Plus, 11 looks a little like
+        # ll (i.e. "Lock Lost").
+        worker.on_current_job_lock_lost { |job| exit!(11) }
+        @modules.each { |mod| worker.extend(mod) }
+        worker
+      end
+
+      # If @sandbox_mutex is free, execute block immediately.
+      # Otherwise, postpone it until handling is possible
+      def contention_aware_handler(&block)
+        if @sandbox_mutex.try_lock
+          block.call
+          @sandbox_mutex.unlock
+        else
+          @postponed_actions_queue << block
+        end
+      end
+
+      # Process any signals (such as TERM) that could not be processed
+      # immediately due to @sandbox_mutex being in use
+      def process_postponed_actions
+        until @postponed_actions_queue.empty?
+          # It's possible a signal interrupteed us between the empty?
+          # and shift calls, but it could have only added more things
+          # into @postponed_actions_queue
+          block = @postponed_actions_queue.shift(true)
+          @sandbox_mutex.synchronize do
+            block.call
+          end
+        end
+      end
+
+      # Register our handling of signals
+      def register_signal_handlers
+        # If we're the parent process, we mostly want to forward the signals on
+        # to the child processes. It's just that sometimes we want to wait for
+        # them and then exit
+        trap('TERM') do
+          contention_aware_handler { stop!('TERM', in_signal_handler=true); exit }
+        end
+        trap('INT') do
+          contention_aware_handler { stop!('INT', in_signal_handler=true); exit }
+        end
+        safe_trap('HUP') { sighup_handler.call }
+        safe_trap('QUIT') do
+          contention_aware_handler { stop!('QUIT', in_signal_handler=true); exit }
+        end
+        safe_trap('USR1') do
+          contention_aware_handler { stop!('KILL', in_signal_handler=true) }
+        end
+        begin
+          trap('CONT') { stop('CONT', in_signal_handler=true) }
+          trap('USR2') { stop('USR2', in_signal_handler=true) }
+        rescue ArgumentError
+          warn 'Signals USR2, and/or CONT not supported.'
+        end
+      end
+
+      # Run this worker
+      def run
+        startup_sandboxes
+
+        # Now keep an eye on our child processes, spawn replacements as needed
+        loop do
+          begin
+            # Don't wait on any processes if we're already in shutdown mode.
+            break if @shutdown
+
+            # Wait for any child to kick the bucket
+            pid, status = Process.wait2
+            code, sig = status.exitstatus, status.stopsig
+            log(:warn,
+              "Worker process #{pid} died with #{code} from signal (#{sig})")
+
+            # allow our shutdown logic (called from a separate thread) to take affect.
+            break if @shutdown
+
+            spawn_replacement_child(pid)
+            process_postponed_actions
+          rescue SystemCallError => e
+            log(:error, "Failed to wait for child process: #{e.inspect}")
+            # If we're shutting down, the loop above will exit
+            exit! unless @shutdown
+          end
+        end
+      end
+
+      # Returns a list of each of the child pids
+      def children
+        @sandboxes.keys
+      end
+
+      # Signal all the children
+      def stop(signal = 'QUIT', in_signal_handler=true)
+        log(:warn, "Sending #{signal} to children") unless in_signal_handler
+        children.each do |pid|
+          begin
+            Process.kill(signal, pid)
+          rescue Errno::ESRCH
+            # no such process -- means the process has already died.
+          end
+        end
+      end
+
+      # Signal all the children and wait for them to exit.
+      # Should only be called when we have the lock on @sandbox_mutex
+      def stop!(signal = 'QUIT', in_signal_handler=true)
+        shutdown(in_signal_handler=in_signal_handler)
+        shutdown_sandboxes(signal, in_signal_handler=in_signal_handler)
+      end
+
+    private
+
+      def startup_sandboxes
+        # Make sure we respond to signals correctly
+        register_signal_handlers
+
+        log(:debug, "Starting to run with #{@num_workers} workers")
+        @num_workers.times do |i|
+          slot = {
+            worker_id: i,
+            sandbox: nil
+          }
+
+          cpid = fork_child_process do
+            # Wait for a bit to calm the thundering herd
+            sleep(rand(max_startup_interval)) if max_startup_interval > 0
+          end
+
+          # If we're the parent process, save information about the child
+          log(:info, "Spawned worker #{cpid}")
+          @sandboxes[cpid] = slot
+        end
+      end
+
+      # Should only be called when we have a lock on @sandbox_mutex
+      def shutdown_sandboxes(signal, in_signal_handler=true)
+        # First, send the signal
+        stop(signal, in_signal_handler=in_signal_handler)
+
+        # Wait for each of our children
+        log(:warn, 'Waiting for child processes') unless in_signal_handler
+
+        until @sandboxes.empty?
+          begin
+            pid, _ = Process.wait2
+            log(:warn, "Child #{pid} stopped") unless in_signal_handler
+            @sandboxes.delete(pid)
+          rescue SystemCallError
+            break
+          end
+        end
+
+
+        unless in_signal_handler
+          log(:warn, 'All children have stopped')
+
+          # If there were any children processes we couldn't wait for, log it
+          @sandboxes.keys.each do |cpid|
+            log(:warn, "Could not wait for child #{cpid}")
+          end
+        end
+
+        @sandboxes.clear
+      end
+
+    private
+
+      def spawn_replacement_child(pid)
+        @sandbox_mutex.synchronize do
+          return if @shutdown
+
+          # And give its slot to a new worker process
+          slot = @sandboxes.delete(pid)
+          cpid = fork_child_process
+
+          # If we're the parent process, ave information about the child
+          log(:warn, "Spawned worker #{cpid} to replace #{pid}")
+          @sandboxes[cpid] = slot
+        end
+      end
+
+      # returns child's pid.
+      def fork_child_process
+        fork do
+          yield if block_given?
+          reconnect_each_client
+          after_fork
+          spawn.run
+        end
+      end
+
+    end
+  end
+end