reqless 0.0.1

data/lib/reqless/throttle.rb

@@ -0,0 +1,57 @@
+# Encoding: utf-8
+
+require 'redis'
+require 'json'
+
+module Reqless
+  class Throttle
+    attr_reader :name, :client
+
+    def initialize(name, client)
+      @name = name
+      @client = client
+    end
+
+    def delete
+      @client.call('throttle.delete', @name)
+    end
+
+    def expiration=(expire_time_in_seconds)
+      update(nil, Integer(expire_time_in_seconds))
+    end
+
+    def id
+      @name
+    end
+
+    def locks
+      JSON.parse(@client.call('throttle.locks', @name))
+    end
+
+    def maximum
+      throttle_attrs['maximum'].to_i
+    end
+
+    def maximum=(max)
+      update(max)
+    end
+
+    def pending
+      JSON.parse(@client.call('throttle.pending', @name))
+    end
+
+    def ttl
+      throttle_attrs['ttl'].to_i
+    end
+
+    private
+    def throttle_attrs
+      throttle_json = @client.call('throttle.get', @name)
+      throttle_json ? JSON.parse(throttle_json) : {}
+    end
+
+    def update(max, expiration = 0)
+      @client.call('throttle.set', @name, max || maximum, expiration)
+    end
+  end
+end

data/lib/reqless/version.rb

@@ -0,0 +1,5 @@
+# Encoding: utf-8
+
+module Reqless
+  VERSION = '0.0.1'
+end

data/lib/reqless/worker/base.rb

@@ -0,0 +1,237 @@
+# Encoding: utf-8
+
+# Standard stuff
+require 'time'
+require 'logger'
+require 'thread'
+
+require 'reqless'
+require 'reqless/subscriber'
+
+module Reqless
+  module Workers
+    JobLockLost = Class.new(StandardError)
+
+    class BaseWorker
+      attr_accessor :output, :reserver, :interval, :paused,
+                    :options, :sighup_handler
+
+      def initialize(reserver, options = {})
+        # Our job reserver and options
+        @reserver = reserver
+        @options = options
+
+        # SIGHUP handler
+        @sighup_handler = options.fetch(:sighup_handler) { lambda { } }
+
+        # Our logger
+        @log = options.fetch(:logger) do
+          @output = options.fetch(:output, $stdout)
+          Logger.new(output).tap do |logger|
+            logger.level = options.fetch(:log_level, Logger::WARN)
+            logger.formatter = options.fetch(:log_formatter) do
+              Proc.new { |severity, datetime, progname, msg| "#{datetime}: #{msg}\n" }
+            end
+          end
+        end
+
+        # The interval for checking for new jobs
+        @interval = options.fetch(:interval, 5.0)
+        @current_job_mutex = Mutex.new
+        @current_job = nil
+
+        # Default behavior when a lock is lost: stop after the current job.
+        on_current_job_lock_lost { shutdown }
+      end
+
+      def log_level
+        @log.level
+      end
+
+      def safe_trap(signal_name, &cblock)
+        begin
+          trap(signal_name, cblock)
+        rescue ArgumentError
+          warn "Signal #{signal_name} not supported."
+        end
+      end
+
+      # The meaning of these signals is meant to closely mirror resque
+      #
+      # TERM: Shutdown immediately, stop processing jobs.
+      #  INT: Shutdown immediately, stop processing jobs.
+      # QUIT: Shutdown after the current job has finished processing.
+      # USR1: Kill the forked children immediately, continue processing jobs.
+      # USR2: Pause after this job
+      # CONT: Start processing jobs again after a USR2
+      #  HUP: Print current stack to log and continue
+      def register_signal_handlers
+        # Otherwise, we want to take the appropriate action
+        trap('TERM') { exit! }
+        trap('INT')  { exit! }
+        safe_trap('HUP') { sighup_handler.call }
+        safe_trap('QUIT') { shutdown }
+        begin
+          trap('CONT') { unpause }
+          trap('USR2') { pause }
+        rescue ArgumentError
+          warn 'Signals USR2, and/or CONT not supported.'
+        end
+      end
+
+      # Return an enumerator to each of the jobs provided by the reserver
+      def jobs
+        return Enumerator.new do |enum|
+          loop do
+            begin
+              job = reserver.reserve
+            rescue Exception => error
+              # We want workers to durably stay up, so we don't want errors
+              # during job reserving (e.g. network timeouts, etc) to kill the
+              # worker.
+              log(:error,
+                "Error reserving job: #{error.class}: #{error.message}")
+            end
+
+            # If we ended up getting a job, yield it. Otherwise, we wait
+            if job.nil?
+              no_job_available
+            else
+              self.current_job = job
+              enum.yield(job)
+              self.current_job = nil
+            end
+
+            break if @shutdown
+          end
+        end
+      end
+
+      # Actually perform the job
+      def perform(job)
+        around_perform(job)
+      rescue JobLockLost
+        log(:warn, "Lost lock for job #{job.jid}")
+      rescue Exception => error
+        fail_job(job, error, caller)
+      else
+        try_complete(job)
+      end
+
+      # Allow middleware modules to be mixed in and override the
+      # definition of around_perform while providing a default
+      # implementation so our code can assume the method is present.
+      module SupportsMiddlewareModules
+        def around_perform(job)
+          job.perform
+        end
+
+        def after_fork
+        end
+      end
+
+      include SupportsMiddlewareModules
+
+      # Stop processing after this job
+      def shutdown
+        @shutdown = true
+      end
+      alias stop! shutdown # so we can call `stop!` regardless of the worker type
+
+      # Pause the worker -- take no more new jobs
+      def pause
+        @paused = true
+        procline "Paused -- #{reserver.description}"
+      end
+
+      # Continue taking new jobs
+      def unpause
+        @paused = false
+      end
+
+      # Set the proceline. Not supported on all systems
+      def procline(value)
+        $0 = "reQless-#{Reqless::VERSION}: #{value} at #{Time.now.iso8601}"
+        log(:debug, $PROGRAM_NAME)
+      end
+
+      # Complete the job unless the worker has already put it into another state
+      # by completing / failing / etc. the job
+      def try_complete(job)
+        job.complete unless job.state_changed?
+      rescue Job::CantCompleteError => e
+        # There's not much we can do here. Complete fails in a few cases:
+        #   - The job is already failed (i.e. by another worker)
+        #   - The job is being worked on by another worker
+        #   - The job has been cancelled
+        #
+        # We don't want to (or are able to) fail the job with this error in
+        # any of these cases, so the best we can do is log the failure.
+        log(:error, "Failed to complete #{job.inspect}: #{e.message}")
+      end
+
+      def fail_job(job, error, worker_backtrace)
+        failure = Reqless.failure_formatter.format(job, error, worker_backtrace)
+        log(:error, "Got #{failure.group} failure from #{job.inspect}\n#{failure.message}" )
+        job.fail(*failure)
+      rescue Job::CantFailError => e
+        # There's not much we can do here. Another worker may have cancelled it,
+        # or we might not own the job, etc. Logging is the best we can do.
+        log(:error, "Failed to fail #{job.inspect}: #{e.message}")
+      end
+
+      def deregister
+        uniq_clients.each do |client|
+          client.deregister_workers(client.worker_name)
+        end
+      end
+
+      def uniq_clients
+        @uniq_clients ||= reserver.queues.map(&:client).uniq
+      end
+
+      def on_current_job_lock_lost(&block)
+        @on_current_job_lock_lost = block
+      end
+
+      def listen_for_lost_lock(job)
+        # Ensure subscribers always has a value
+        subscriber = Subscriber.start(job.client, "ql:w:#{job.client.worker_name}", log: @log) do |_, message|
+          if message['event'] == 'lock_lost' && message['jid'] == job.jid
+            @on_current_job_lock_lost.call(job)
+          end
+        end
+
+        yield
+      ensure
+        subscriber && subscriber.stop
+      end
+
+    private
+
+      def log(type, msg)
+        @log.public_send(type, "#{Process.pid}: #{msg}")
+      end
+
+      def no_job_available
+        unless interval.zero?
+          procline "Waiting for #{reserver.description}"
+          log(:debug, "Sleeping for #{interval} seconds")
+          sleep interval
+        end
+      end
+
+      def with_current_job
+        @current_job_mutex.synchronize do
+          yield @current_job
+        end
+      end
+
+      def current_job=(job)
+        @current_job_mutex.synchronize do
+          @current_job = job
+        end
+      end
+    end
+  end
+end

data/lib/reqless/worker/forking.rb

@@ -0,0 +1,215 @@
+# Encoding: utf-8
+
+require 'reqless'
+require 'reqless/worker/base'
+require 'reqless/worker/serial'
+require 'thread'
+
+module Reqless
+  module Workers
+    class ForkingWorker < BaseWorker
+      # The child startup interval
+      attr_accessor :max_startup_interval
+
+      def initialize(reserver, options = {})
+        super(reserver, options)
+        # The keys are the child PIDs, the values are information about the
+        # worker, including its sandbox directory. This directory currently
+        # isn't used, but this sets up for having that eventually.
+        @sandboxes = {}
+
+        # Save our options for starting children
+        @options = options
+
+        # The max interval between when children start (reduces thundering herd)
+        @max_startup_interval = options[:max_startup_interval] || 10.0
+
+        # TODO: facter to figure out how many cores we have
+        @num_workers = options[:num_workers] || 1
+
+        # All the modules that have been applied to this worker
+        @modules = []
+
+        @sandbox_mutex = Mutex.new
+      end
+
+      # Because we spawn a new worker, we need to apply all the modules that
+      # extend this one
+      def extend(mod)
+        @modules << mod
+        super(mod)
+      end
+
+      # Spawn a new child worker
+      def spawn
+        worker = SerialWorker.new(reserver, @options)
+        # We use 11 as the exit status so that it is something unique
+        # (rather than the common 1). Plus, 11 looks a little like
+        # ll (i.e. "Lock Lost").
+        worker.on_current_job_lock_lost { |job| exit!(11) }
+        @modules.each { |mod| worker.extend(mod) }
+        worker
+      end
+
+      # Register our handling of signals
+      def register_signal_handlers
+        # If we're the parent process, we mostly want to forward the signals on
+        # to the child processes. It's just that sometimes we want to wait for
+        # them and then exit
+        trap('TERM') do
+          stop!('TERM')
+          exit
+        end
+
+        trap('INT') do
+          stop!('TERM')
+          exit
+        end
+
+        safe_trap('HUP') { sighup_handler.call }
+        safe_trap('QUIT') do
+          stop!('QUIT')
+          exit
+        end
+        safe_trap('USR1') { stop!('KILL') }
+
+        begin
+          trap('CONT') { stop('CONT') }
+          trap('USR2') { stop('USR2') }
+        rescue ArgumentError
+          warn 'Signals USR2, and/or CONT not supported.'
+        end
+      end
+
+      # Run this worker
+      def run
+        startup_sandboxes
+
+        # Now keep an eye on our child processes, spawn replacements as needed
+        loop do
+          begin
+            # Don't wait on any processes if we're already in shutdown mode.
+            break if @shutdown
+
+            # Wait for any child to kick the bucket
+            pid, status = Process.wait2
+            code, sig = status.exitstatus, status.stopsig
+            log(:warn,
+              "Worker process #{pid} died with #{code} from signal (#{sig})")
+
+            # allow our shutdown logic (called from a separate thread) to take affect.
+            break if @shutdown
+
+            spawn_replacement_child(pid)
+          rescue SystemCallError => e
+            log(:error, "Failed to wait for child process: #{e.inspect}")
+            # If we're shutting down, the loop above will exit
+            exit! unless @shutdown
+          end
+        end
+      end
+
+      # Returns a list of each of the child pids
+      def children
+        @sandboxes.keys
+      end
+
+      # Signal all the children
+      def stop(signal = 'QUIT')
+        log(:warn, "Sending #{signal} to children")
+        children.each do |pid|
+          begin
+            Process.kill(signal, pid)
+          rescue Errno::ESRCH
+            # no such process -- means the process has already died.
+          end
+        end
+      end
+
+      # Signal all the children and wait for them to exit
+      def stop!(signal = 'QUIT')
+        shutdown
+        shutdown_sandboxes(signal)
+      end
+
+    private
+
+      def startup_sandboxes
+        # Make sure we respond to signals correctly
+        register_signal_handlers
+
+        log(:debug, "Starting to run with #{@num_workers} workers")
+        @num_workers.times do |i|
+          slot = {
+            worker_id: i,
+            sandbox: nil
+          }
+
+          cpid = fork_child_process do
+            # Wait for a bit to calm the thundering herd
+            sleep(rand(max_startup_interval)) if max_startup_interval > 0
+          end
+
+          # If we're the parent process, save information about the child
+          log(:info, "Spawned worker #{cpid}")
+          @sandboxes[cpid] = slot
+        end
+      end
+
+      def shutdown_sandboxes(signal)
+        @sandbox_mutex.synchronize do
+          # First, send the signal
+          stop(signal)
+
+          # Wait for each of our children
+          log(:warn, 'Waiting for child processes')
+
+          until @sandboxes.empty?
+            begin
+              pid, _ = Process.wait2
+              log(:warn, "Child #{pid} stopped")
+              @sandboxes.delete(pid)
+            rescue SystemCallError
+              break
+            end
+          end
+
+          log(:warn, 'All children have stopped')
+
+          # If there were any children processes we couldn't wait for, log it
+          @sandboxes.keys.each do |cpid|
+            log(:warn, "Could not wait for child #{cpid}")
+          end
+
+          @sandboxes.clear
+        end
+      end
+
+    private
+
+      def spawn_replacement_child(pid)
+        @sandbox_mutex.synchronize do
+          return if @shutdown
+
+          # And give its slot to a new worker process
+          slot = @sandboxes.delete(pid)
+          cpid = fork_child_process
+
+          # If we're the parent process, ave information about the child
+          log(:warn, "Spawned worker #{cpid} to replace #{pid}")
+          @sandboxes[cpid] = slot
+        end
+      end
+
+      # returns child's pid.
+      def fork_child_process
+        fork do
+          yield if block_given?
+          after_fork
+          spawn.run
+        end
+      end
+
+    end
+  end
+end

data/lib/reqless/worker/serial.rb

@@ -0,0 +1,41 @@
+# Encoding: utf-8
+
+require 'reqless'
+require 'reqless/worker/base'
+
+module Reqless
+  module Workers
+    # A worker that keeps popping off jobs and processing them
+    class SerialWorker < BaseWorker
+      def initialize(reserver, options = {})
+        super(reserver, options)
+      end
+
+      def run
+        log(:info, "Starting #{reserver.description} in #{Process.pid}")
+        procline "Starting #{reserver.description}"
+        register_signal_handlers
+
+        reserver.prep_for_work!
+
+        procline "Running #{reserver.description}"
+
+        jobs.each do |job|
+          # Run the job we're working on
+          log(:debug, "Starting job #{job.klass_name} (#{job.jid} from #{job.queue_name})")
+          procline "Processing #{job.description}"
+          listen_for_lost_lock(job) do
+            perform(job)
+          end
+          log(:debug, "Finished job #{job.klass_name} (#{job.jid} from #{job.queue_name})")
+
+          # So long as we're paused, we should wait
+          while paused
+            log(:debug, 'Paused...')
+            sleep interval
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reqless/worker.rb

@@ -0,0 +1,5 @@
+# Encoding: utf-8
+
+require 'reqless/worker/base'
+require 'reqless/worker/serial'
+require 'reqless/worker/forking'