qs 0.0.1 → 0.1.0

data/lib/qs/payload_handler.rb

@@ -0,0 +1,136 @@
+require 'benchmark'
+require 'dat-worker-pool'
+require 'qs'
+require 'qs/error_handler'
+require 'qs/job'
+require 'qs/logger'
+
+module Qs
+
+  class PayloadHandler
+
+    attr_reader :daemon_data, :redis_item, :logger
+
+    def initialize(daemon_data, redis_item)
+      @daemon_data = daemon_data
+      @redis_item  = redis_item
+      @logger = Qs::Logger.new(
+        @daemon_data.logger,
+        @daemon_data.verbose_logging
+      )
+    end
+
+    def run
+      log_received
+      benchmark = Benchmark.measure{ run!(@daemon_data, @redis_item) }
+      @redis_item.time_taken = RoundedTime.new(benchmark.real)
+      log_complete(@redis_item)
+      raise_if_debugging!(@redis_item.exception)
+    end
+
+    private
+
+    def run!(daemon_data, redis_item)
+      redis_item.started = true
+
+      payload = Qs.deserialize(redis_item.serialized_payload)
+      job = Qs::Job.parse(payload)
+      log_job(job)
+      redis_item.job = job
+
+      route = daemon_data.route_for(job.name)
+      log_handler_class(route.handler_class)
+      redis_item.handler_class = route.handler_class
+
+      route.run(job, daemon_data)
+      redis_item.finished = true
+    rescue DatWorkerPool::ShutdownError => exception
+      if redis_item.started
+        error = ShutdownError.new(exception.message)
+        error.set_backtrace(exception.backtrace)
+        handle_exception(error, daemon_data, redis_item)
+      end
+      raise exception
+    rescue StandardError => exception
+      handle_exception(exception, daemon_data, redis_item)
+    end
+
+    def handle_exception(exception, daemon_data, redis_item)
+      error_handler = Qs::ErrorHandler.new(exception, {
+        :daemon_data        => daemon_data,
+        :queue_redis_key    => redis_item.queue_redis_key,
+        :serialized_payload => redis_item.serialized_payload,
+        :job                => redis_item.job,
+        :handler_class      => redis_item.handler_class
+      }).tap(&:run)
+      redis_item.exception = error_handler.exception
+      log_exception(redis_item.exception)
+    end
+
+    def raise_if_debugging!(exception)
+      raise exception if exception && ENV['QS_DEBUG']
+    end
+
+    def log_received
+      log_verbose "===== Running job ====="
+    end
+
+    def log_job(job)
+      log_verbose "  Job:     #{job.name.inspect}"
+      log_verbose "  Params:  #{job.params.inspect}"
+    end
+
+    def log_handler_class(handler_class)
+      log_verbose "  Handler: #{handler_class}"
+    end
+
+    def log_complete(redis_item)
+      log_verbose "===== Completed in #{redis_item.time_taken}ms ====="
+      summary_line_args = {
+        'time'    => redis_item.time_taken,
+        'handler' => redis_item.handler_class
+      }
+      if (job = redis_item.job)
+        summary_line_args['job']    = job.name
+        summary_line_args['params'] = job.params
+      end
+      if (exception = redis_item.exception)
+        summary_line_args['error'] = "#{exception.inspect}"
+      end
+      log_summary SummaryLine.new(summary_line_args)
+    end
+
+    def log_exception(exception)
+      backtrace = exception.backtrace.join("\n")
+      message = "#{exception.class}: #{exception.message}\n#{backtrace}"
+      log_verbose(message, :error)
+    end
+
+    def log_verbose(message, level = :info)
+      self.logger.verbose.send(level, "[Qs] #{message}")
+    end
+
+    def log_summary(message, level = :info)
+      self.logger.summary.send(level, "[Qs] #{message}")
+    end
+
+    module RoundedTime
+      ROUND_PRECISION = 2
+      ROUND_MODIFIER = 10 ** ROUND_PRECISION
+      def self.new(time_in_seconds)
+        (time_in_seconds * 1000 * ROUND_MODIFIER).to_i / ROUND_MODIFIER.to_f
+      end
+    end
+
+    module SummaryLine
+      def self.new(line_attrs)
+        attr_keys = %w{time handler job params error}
+        attr_keys.map{ |k| "#{k}=#{line_attrs[k].inspect}" }.join(' ')
+      end
+    end
+
+  end
+
+  ShutdownError = Class.new(DatWorkerPool::ShutdownError)
+
+end

data/lib/qs/pid_file.rb

@@ -0,0 +1,42 @@
+require 'fileutils'
+
+module Qs
+
+  class PIDFile
+    attr_reader :path
+
+    def initialize(path)
+      @path = (path || '/dev/null').to_s
+    end
+
+    def pid
+      pid = File.read(@path).strip
+      pid && !pid.empty? ? pid.to_i : raise('no pid in file')
+    rescue StandardError => exception
+      error = InvalidError.new("A PID couldn't be read from #{@path.inspect}")
+      error.set_backtrace(exception.backtrace)
+      raise error
+    end
+
+    def write
+      FileUtils.mkdir_p(File.dirname(@path))
+      File.open(@path, 'w'){ |f| f.puts ::Process.pid }
+    rescue StandardError => exception
+      error = InvalidError.new("Can't write pid to file #{@path.inspect}")
+      error.set_backtrace(exception.backtrace)
+      raise error
+    end
+
+    def remove
+      FileUtils.rm_f(@path)
+    end
+
+    def to_s
+      @path
+    end
+
+    InvalidError = Class.new(RuntimeError)
+
+  end
+
+end

data/lib/qs/process.rb

@@ -0,0 +1,136 @@
+require 'qs/pid_file'
+
+module Qs
+
+  class Process
+
+    attr_reader :daemon, :name, :pid_file, :restart_cmd
+
+    def initialize(daemon, options = nil)
+      options ||= {}
+      @daemon = daemon
+      @logger = @daemon.logger
+      @pid_file = PIDFile.new(@daemon.pid_file)
+      @restart_cmd = RestartCmd.new
+
+      @name = ignore_if_blank(ENV['QS_PROCESS_NAME']) || "qs-#{@daemon.name}"
+
+      @daemonize = !!options[:daemonize]
+      @skip_daemonize = !!ignore_if_blank(ENV['QS_SKIP_DAEMONIZE'])
+      @restart = false
+    end
+
+    def run
+      ::Process.daemon(true) if self.daemonize?
+      log "Starting Qs daemon for #{@daemon.name}..."
+
+      $0 = @name
+      @pid_file.write
+      log "PID: #{@pid_file.pid}"
+
+      ::Signal.trap("TERM"){ @daemon.stop }
+      ::Signal.trap("INT"){ @daemon.halt }
+      ::Signal.trap("USR2") do
+        @daemon.stop
+        @restart = true
+      end
+
+      thread = @daemon.start
+      log "#{@daemon.name} daemon started and ready."
+      thread.join
+      run_restart_cmd if self.restart?
+    rescue StandardError => exception
+      log "Error: #{exception.message}"
+      log "#{@daemon.name} daemon never started."
+    ensure
+      @pid_file.remove
+    end
+
+    def daemonize?
+      @daemonize && !@skip_daemonize
+    end
+
+    def restart?
+      @restart
+    end
+
+    private
+
+    def log(message)
+      @logger.info "[Qs] #{message}"
+    end
+
+    def run_restart_cmd
+      log "Restarting #{@daemon.name} daemon..."
+      ENV['QS_SKIP_DAEMONIZE'] = 'yes'
+      @restart_cmd.run
+    end
+
+    def default_if_blank(value, default, &block)
+      ignore_if_blank(value, &block) || default
+    end
+
+    def ignore_if_blank(value, &block)
+      block ||= proc{ |v| v }
+      block.call(value) if value && !value.empty?
+    end
+
+  end
+
+  class RestartCmd
+    attr_reader :argv, :dir
+
+    def initialize
+      require 'rubygems'
+      @dir  = get_pwd
+      @argv = [Gem.ruby, $0, ARGV.dup].flatten
+    end
+
+    def run
+      Dir.chdir self.dir
+      Kernel.exec(*self.argv)
+    end
+
+    private
+
+    # Trick from puma/unicorn. Favor PWD because it contains an unresolved
+    # symlink. This is useful when restarting after deploying; the original
+    # directory may be removed, but the symlink is pointing to a new
+    # directory.
+    def get_pwd
+      return Dir.pwd if ENV['PWD'].nil?
+      env_stat = File.stat(ENV['PWD'])
+      pwd_stat = File.stat(Dir.pwd)
+      if env_stat.ino == pwd_stat.ino && env_stat.dev == pwd_stat.dev
+        ENV['PWD']
+      else
+        Dir.pwd
+      end
+    end
+  end
+
+  # This is from puma for 1.8 compatibility. Ruby 1.9+ defines a
+  # `Process.daemon` for daemonizing processes. This defines the method when it
+  # isn't provided, i.e. Ruby 1.8.
+  unless ::Process.respond_to?(:daemon)
+    ::Process.class_eval do
+
+      # Full explanation: http://www.steve.org.uk/Reference/Unix/faq_2.html#SEC16
+      def self.daemon(no_chdir = false, no_close = false)
+        exit if fork
+        ::Process.setsid
+        exit if fork
+        Dir.chdir '/' unless no_chdir
+        if !no_close
+          null = File.open('/dev/null', 'w')
+          STDIN.reopen null
+          STDOUT.reopen null
+          STDERR.reopen null
+        end
+        return 0
+      end
+
+    end
+  end
+
+end

data/lib/qs/process_signal.rb

@@ -0,0 +1,20 @@
+require 'qs/pid_file'
+
+module Qs
+
+  class ProcessSignal
+
+    attr_reader :signal, :pid
+
+    def initialize(daemon, signal)
+      @signal = signal
+      @pid = PIDFile.new(daemon.pid_file).pid
+    end
+
+    def send
+      ::Process.kill(@signal, @pid)
+    end
+
+  end
+
+end

data/lib/qs/qs_runner.rb

@@ -0,0 +1,49 @@
+require 'system_timer'
+require 'qs'
+require 'qs/runner'
+
+module Qs
+
+  class QsRunner < Runner
+
+    attr_reader :timeout
+
+    def initialize(handler_class, args = nil)
+      super(handler_class, args)
+      @timeout = handler_class.timeout || Qs.config.timeout
+    end
+
+    def run
+      OptionalTimeout.new(self.timeout) do
+        run_callbacks self.handler_class.before_callbacks
+        self.handler.init
+        self.handler.run
+        run_callbacks self.handler_class.after_callbacks
+      end
+    rescue TimeoutError => exception
+      error = TimeoutError.new "#{handler_class} timed out (#{timeout}s)"
+      error.set_backtrace(exception.backtrace)
+      raise error
+    end
+
+    private
+
+    def run_callbacks(callbacks)
+      callbacks.each{ |proc| self.handler.instance_eval(&proc) }
+    end
+
+    module OptionalTimeout
+      def self.new(timeout, &block)
+        if !timeout.nil?
+          SystemTimer.timeout_after(timeout, TimeoutError, &block)
+        else
+          block.call
+        end
+      end
+    end
+
+  end
+
+  TimeoutError = Class.new(RuntimeError)
+
+end

data/lib/qs/queue.rb

@@ -0,0 +1,69 @@
+require 'qs/route'
+
+module Qs
+
+  class Queue
+
+    attr_reader :routes
+    attr_reader :enqueued_jobs
+
+    def initialize(&block)
+      @routes = []
+      @enqueued_jobs = []
+      self.instance_eval(&block) if !block.nil?
+      raise InvalidError, "a queue must have a name" if self.name.nil?
+    end
+
+    def name(value = nil)
+      @name = value if !value.nil?
+      @name
+    end
+
+    def redis_key
+      @redis_key ||= RedisKey.new(self.name)
+    end
+
+    def job_handler_ns(value = nil)
+      @job_handler_ns = value if !value.nil?
+      @job_handler_ns
+    end
+
+    def job(name, handler_name)
+      if self.job_handler_ns && !(handler_name =~ /^::/)
+        handler_name = "#{self.job_handler_ns}::#{handler_name}"
+      end
+
+      @routes.push(Qs::Route.new(name, handler_name))
+    end
+
+    def enqueue(job_name, params = nil)
+      Qs.enqueue(self, job_name, params)
+    end
+    alias :add :enqueue
+
+    def reset!
+      self.enqueued_jobs.clear
+    end
+
+    def inspect
+      reference = '0x0%x' % (self.object_id << 1)
+      "#<#{self.class}:#{reference} " \
+        "@name=#{self.name.inspect} " \
+        "@job_handler_ns=#{self.job_handler_ns.inspect}>"
+    end
+
+    InvalidError = Class.new(RuntimeError)
+
+    module RedisKey
+      def self.parse_name(key)
+        key.split(':').last
+      end
+
+      def self.new(name)
+        "queues:#{name}"
+      end
+    end
+
+  end
+
+end

data/lib/qs/redis_item.rb

@@ -0,0 +1,33 @@
+module Qs
+
+  class RedisItem
+
+    attr_reader :queue_redis_key, :serialized_payload
+    attr_accessor :started, :finished
+    attr_accessor :job, :handler_class
+    attr_accessor :exception, :time_taken
+
+    def initialize(queue_redis_key, serialized_payload)
+      @queue_redis_key    = queue_redis_key
+      @serialized_payload = serialized_payload
+      @started            = false
+      @finished           = false
+
+      @job           = nil
+      @handler_class = nil
+      @exception     = nil
+      @time_taken    = nil
+    end
+
+    def ==(other)
+      if other.kind_of?(self.class)
+        self.queue_redis_key    == other.queue_redis_key &&
+        self.serialized_payload == other.serialized_payload
+      else
+        super
+      end
+    end
+
+  end
+
+end

data/lib/qs/route.rb

@@ -0,0 +1,52 @@
+require 'qs/qs_runner'
+
+module Qs
+
+  class Route
+
+    attr_reader :name, :handler_class_name, :handler_class
+
+    def initialize(name, handler_class_name)
+      @name = name.to_s
+      @handler_class_name = handler_class_name
+      @handler_class = nil
+    end
+
+    def validate!
+      @handler_class = constantize_handler_class(@handler_class_name)
+    end
+
+    def run(job, daemon_data)
+      QsRunner.new(self.handler_class, {
+        :job    => job,
+        :params => job.params,
+        :logger => daemon_data.logger
+      }).run
+    end
+
+    private
+
+    def constantize_handler_class(handler_class_name)
+      constantize(handler_class_name).tap do |handler_class|
+        raise(NoHandlerClassError.new(handler_class_name)) if !handler_class
+      end
+    end
+
+    def constantize(class_name)
+      names = class_name.to_s.split('::').reject{ |name| name.empty? }
+      klass = names.inject(Object){ |constant, name| constant.const_get(name) }
+      klass == Object ? false : klass
+    rescue NameError
+      false
+    end
+
+  end
+
+  class NoHandlerClassError < RuntimeError
+    def initialize(handler_class_name)
+      super "Qs couldn't find the handler '#{handler_class_name}'" \
+            " - it doesn't exist or hasn't been required in yet."
+    end
+  end
+
+end

data/lib/qs/runner.rb

@@ -0,0 +1,26 @@
+require 'qs/logger'
+
+module Qs
+
+  class Runner
+
+    attr_reader :handler_class, :handler
+    attr_reader :job, :params, :logger
+
+    def initialize(handler_class, args = nil)
+      @handler_class = handler_class
+      @handler = @handler_class.new(self)
+
+      a = args || {}
+      @job    = a[:job]
+      @params = a[:params] || {}
+      @logger = a[:logger] || Qs::NullLogger.new
+    end
+
+    def run
+      raise NotImplementedError
+    end
+
+  end
+
+end

data/lib/qs/test_helpers.rb

@@ -0,0 +1,17 @@
+require 'qs/test_runner'
+
+module Qs
+
+  module TestHelpers
+
+    def test_runner(handler_class, args = nil)
+      TestRunner.new(handler_class, args)
+    end
+
+    def test_handler(handler_class, args = nil)
+      test_runner(handler_class, args).handler
+    end
+
+  end
+
+end

data/lib/qs/test_runner.rb

@@ -0,0 +1,43 @@
+require 'qs'
+require 'qs/job'
+require 'qs/job_handler'
+require 'qs/runner'
+
+module Qs
+
+  InvalidJobHandlerError = Class.new(StandardError)
+
+  class TestRunner < Runner
+
+    def initialize(handler_class, args = nil)
+      if !handler_class.include?(Qs::JobHandler)
+        raise InvalidJobHandlerError, "#{handler_class.inspect} is not a"\
+                                      " Qs::JobHandler"
+      end
+      args = (args || {}).dup
+      super(handler_class, {
+        :job    => args.delete(:job),
+        :params => normalize_params(args.delete(:params) || {}),
+        :logger => args.delete(:logger)
+      })
+      args.each{ |key, value| self.handler.send("#{key}=", value) }
+
+      self.handler.init
+    end
+
+    def run
+      self.handler.run
+    end
+
+    private
+
+    # Stringify and serialize/deserialize to ensure params are valid and are
+    # in the format they would normally be when a handler is built and run.
+    def normalize_params(params)
+      params = Job::StringifyParams.new(params)
+      Qs.deserialize(Qs.serialize(params))
+    end
+
+  end
+
+end

data/lib/qs/version.rb

@@ -1,3 +1,3 @@
 module Qs
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end