qless 0.9.3 → 0.10.0

data/lib/qless/lua_script.rb

@@ -1,8 +1,13 @@
+# Encoding: utf-8
+
 require 'digest/sha1'
 
 module Qless
+  LuaScriptError = Class.new(Qless::Error)
+
+  # Wraps a lua script. Knows how to reload it if necessary
   class LuaScript
-    LUA_SCRIPT_DIR = File.expand_path("../qless-core/", __FILE__)
+    SCRIPT_ROOT = File.expand_path('../lua', __FILE__)
 
     def initialize(name, redis)
       @name  = name
@@ -12,31 +17,77 @@ module Qless
 
     attr_reader :name, :redis, :sha
 
-    def reload()
+    def reload
       @sha = @redis.script(:load, script_contents)
     end
 
-    def call(keys, argv)
-      _call(keys, argv)
-    rescue
-      reload
-      _call(keys, argv)
+    def call(*argv)
+      handle_no_script_error do
+        _call(*argv)
+      end
+    rescue Redis::CommandError => err
+      if match = err.message.match('user_script:\d+:\s*(\w+.+$)')
+        raise LuaScriptError.new(match[1])
+      else
+        raise err
+      end
     end
 
   private
 
     if USING_LEGACY_REDIS_VERSION
-      def _call(keys, argv)
-        @redis.evalsha(@sha, keys.length, *(keys + argv))
+      def _call(*argv)
+        @redis.evalsha(@sha, 0, *argv)
       end
     else
-      def _call(keys, argv)
-        @redis.evalsha(@sha, keys: keys, argv: argv)
+      def _call(*argv)
+        @redis.evalsha(@sha, keys: [], argv: argv)
+      end
+    end
+
+    def handle_no_script_error
+      yield
+    rescue ScriptNotLoadedRedisCommandError
+      reload
+      yield
+    end
+
+    # Module for notifying when a script hasn't yet been loaded
+    module ScriptNotLoadedRedisCommandError
+      MESSAGE = 'NOSCRIPT No matching script. Please use EVAL.'
+
+      def self.===(error)
+        error.is_a?(Redis::CommandError) && error.message == MESSAGE
       end
     end
 
     def script_contents
-      @script_contents ||= File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua"))
+      @script_contents ||= File.read(File.join(SCRIPT_ROOT, "#{@name}.lua"))
     end
   end
+
+  # Provides a simple way to load and use lua-based Qless plugins.
+  # This combines the qless-lib.lua script plus your custom script
+  # contents all into one script, so that your script can use
+  # Qless's lua API.
+  class LuaPlugin < LuaScript
+    def initialize(name, redis, plugin_contents)
+      @name  = name
+      @redis = redis
+      @plugin_contents = plugin_contents.gsub(COMMENT_LINES_RE, '')
+      super(name, redis)
+    end
+
+  private
+
+    def script_contents
+      @script_contents ||= [QLESS_LIB_CONTENTS, @plugin_contents].join("\n\n")
+    end
+
+    COMMENT_LINES_RE = /^\s*--.*$\n?/
+
+    QLESS_LIB_CONTENTS = File.read(
+      File.join(SCRIPT_ROOT, 'qless-lib.lua')
+    ).gsub(COMMENT_LINES_RE, '')
+  end
 end

data/lib/qless/middleware/memory_usage_monitor.rb

@@ -0,0 +1,62 @@
+
+module Qless
+  module Middleware
+    # Monitors the memory usage of the Qless worker, instructing
+    # it to shutdown when memory exceeds the given :max_memory threshold.
+    class MemoryUsageMonitor < Module
+      def initialize(options)
+        max_memory = options.fetch(:max_memory)
+
+        module_eval do
+          job_counter = 0
+
+          define_method :around_perform do |job|
+            job_counter += 1
+
+            begin
+              super(job)
+            ensure
+              current_mem = MemoryUsageMonitor.current_usage_in_kb
+              if current_mem > max_memory
+                log(:info, "Exiting after job #{job_counter} since current memory " \
+                           "(#{current_mem} KB) has exceeded max allowed memory " \
+                           "(#{max_memory} KB).")
+                shutdown
+              end
+            end
+          end
+        end
+      end
+
+      SHELL_OUT_FOR_MEMORY = -> do
+        # Taken from:
+        # http://stackoverflow.com/a/4133642/29262
+        Integer(`ps -o rss= -p #{Process.pid}`)
+      end unless defined?(SHELL_OUT_FOR_MEMORY)
+
+      begin
+        require 'rusage'
+      rescue LoadError
+        warn "Could not load `rusage` gem. Falling back to shelling out "
+             "to get process memory usage, which is several orders of magnitude slower."
+
+        define_singleton_method(:current_usage_in_kb, &SHELL_OUT_FOR_MEMORY)
+      else
+        memory_ratio = Process.rusage.maxrss / SHELL_OUT_FOR_MEMORY.().to_f
+
+        if (800...1200).cover?(memory_ratio)
+          # OS X tends to return maxrss in Bytes.
+          def self.current_usage_in_kb
+            Process.rusage.maxrss / 1024
+          end
+        else
+          # Linux tends to return maxrss in KB.
+          def self.current_usage_in_kb
+            Process.rusage.maxrss
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/qless/middleware/metriks.rb

@@ -0,0 +1,45 @@
+# Encoding: utf-8
+
+require 'metriks'
+
+module Qless
+  module Middleware
+    module Metriks
+
+      # Tracks the time jobs take, grouping the timings by the job class.
+      module TimeJobsByClass
+        def around_perform(job)
+          ::Metriks.timer("qless.job-times.#{job.klass_name}").time do
+            super
+          end
+        end
+      end
+
+      # Increments a counter each time an instance of a particular job class
+      # completes.
+      #
+      # Usage:
+      #
+      # Qless::Worker.class_eval do
+      #   include Qless::Middleware::CountEvents.new(
+      #     SomeJobClass => "event_name",
+      #     SomeOtherJobClass => "some_other_event"
+      #   )
+      # end
+      class CountEvents < Module
+        def initialize(class_to_event_map)
+          module_eval do # eval the block within the module instance
+            define_method :around_perform do |job|
+              super(job)
+              return unless job.state == 'complete'
+              return unless event_name = class_to_event_map[job.klass]
+
+              counter = ::Metriks.counter("qless.job-events.#{event_name}")
+              counter.increment
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/qless/middleware/redis_reconnect.rb

@@ -1,13 +1,17 @@
+# Encoding: utf-8
+
 module Qless
   module Middleware
+    # A module for reconnecting to redis for each job
     module RedisReconnect
       def self.new(*redis_connections, &block)
         Module.new do
           define_singleton_method :to_s do
-            "Qless::Middleware::RedisReconnect"
+            'Qless::Middleware::RedisReconnect'
           end
+          define_singleton_method(:inspect, method(:to_s))
 
-          block ||= lambda { |job| redis_connections }
+          block ||= ->(job) { redis_connections }
 
           define_method :around_perform do |job|
             Array(block.call(job)).each do |redis|
@@ -21,4 +25,3 @@ module Qless
     end
   end
 end
-

data/lib/qless/middleware/requeue_exceptions.rb

@@ -0,0 +1,94 @@
+# Encoding: utf-8
+
+module Qless
+  module Middleware
+    # This middleware is like RetryExceptions, but it doesn't use qless-core's
+    # internal retry/retry-tracking mechanism. Instead, it re-queues the job
+    # when it fails with a matched error, and increments a counter in the job's
+    # data.
+    #
+    # This is useful for exceptions for which you want a different
+    # backoff/retry strategy. The internal retry mechanism doesn't allow for
+    # separate tracking by exception type, and thus doesn't allow you to retry
+    # different exceptions a different number of times.
+    #
+    # This is particularly useful for handling resource throttling errors,
+    # where you may not want exponential backoff, and you may want the error
+    # to be retried many times, w/o having other transient errors retried so
+    # many times.
+    module RequeueExceptions
+      RequeueableException = Struct.new(:klass, :delay_min, :delay_span, :max_attempts) do
+        def self.from_splat_and_options(*klasses, options)
+          delay_range = options.fetch(:delay_range)
+          delay_min = Float(delay_range.min)
+          delay_span = Float(delay_range.max) - Float(delay_range.min)
+          max_attempts = options.fetch(:max_attempts)
+          klasses.map do |klass|
+            new(klass, delay_min, delay_span, max_attempts)
+          end
+        end
+
+        def delay
+          delay_min + Random.rand(delay_span)
+        end
+
+        def raise_if_exhausted_requeues(error, requeues)
+          raise error if requeues >= max_attempts
+        end
+      end
+
+      def requeue_on(*exceptions, options)
+        RequeueableException.from_splat_and_options(
+          *exceptions, options).each do |exc|
+          requeueable_exceptions[exc.klass] = exc
+        end
+      end
+
+      DEFAULT_ON_REQUEUE_CALLBACK = lambda { |error, job| }
+      def use_on_requeue_callback(&block)
+        @on_requeue_callback = block if block
+      end
+
+      def on_requeue_callback
+        @on_requeue_callback ||= DEFAULT_ON_REQUEUE_CALLBACK
+      end
+
+      def handle_exception(job, error)
+        config = requeuable_exception_for(error)
+
+        requeues_by_exception = (job.data['requeues_by_exception'] ||= {})
+        requeues_by_exception[config.klass.name] ||= 0
+
+        config.raise_if_exhausted_requeues(
+          error, requeues_by_exception[config.klass.name])
+
+        requeues_by_exception[config.klass.name] += 1
+        job.requeue(job.queue_name, delay: config.delay, data: job.data)
+
+        on_requeue_callback.call(error, job)
+      end
+
+      def around_perform(job)
+        super
+      rescue *requeueable_exceptions.keys => e
+        handle_exception(job, e)
+      end
+
+      def requeueable?(exception)
+        requeueable_exceptions.member?(exception)
+      end
+
+      def requeueable_exceptions
+        @requeueable_exceptions ||= {}
+      end
+
+      def requeuable_exception_for(e)
+        requeueable_exceptions.fetch(e.class) do
+          requeueable_exceptions.each do |klass, exc|
+            break exc if klass === e
+          end
+        end
+      end
+    end
+  end
+end

data/lib/qless/middleware/retry_exceptions.rb

@@ -1,13 +1,24 @@
+# Encoding: utf-8
+
 module Qless
   module Middleware
+    # Auto-retries particular errors using qless-core's internal retry tracking
+    # mechanism. Supports a backoff strategy (typically exponential).
+    #
+    # Note: this does not support varying the number of allowed retries by
+    # exception type. If you want that kind of flexibility, use the
+    # RequeueExceptions middleware instead.
     module RetryExceptions
       def around_perform(job)
         super
-      rescue *retryable_exception_classes => e
+      rescue *retryable_exception_classes => error
         raise if job.retries_left <= 0
 
         attempt_num = (job.original_retries - job.retries_left) + 1
-        job.retry(backoff_strategy.call(attempt_num))
+        failure = Qless.failure_formatter.format(job, error)
+        job.retry(backoff_strategy.call(attempt_num, error), *failure)
+
+        on_retry_callback.call(error, job)
       end
 
       def retryable_exception_classes
@@ -18,7 +29,7 @@ module Qless
         retryable_exception_classes.push(*exception_classes)
       end
 
-      NO_BACKOFF_STRATEGY = lambda { |num| 0 }
+      NO_BACKOFF_STRATEGY = ->(_num, _error) { 0 }
 
       def use_backoff_strategy(strategy = nil, &block)
         @backoff_strategy = strategy || block
@@ -28,16 +39,34 @@ module Qless
         @backoff_strategy ||= NO_BACKOFF_STRATEGY
       end
 
-      def exponential(base, options = {})
-        rand_fuzz = options.fetch(:rand_fuzz, 1)
-        lambda do |num|
-          base ** num + rand(rand_fuzz)
+      DEFAULT_ON_RETRY_CALLBACK = lambda { |error, job| }
+      def use_on_retry_callback(&block)
+        @on_retry_callback = block if block
+      end
+
+      def on_retry_callback
+        @on_retry_callback ||= DEFAULT_ON_RETRY_CALLBACK
+      end
+
+      # If `factor` is omitted it is set to `delay_seconds` to reproduce legacy
+      # behavior.
+      def exponential(delay_seconds, options={})
+        factor = options.fetch(:factor, delay_seconds)
+        fuzz_factor = options.fetch(:fuzz_factor, 0)
+
+        lambda do |retry_no, error|
+          unfuzzed = delay_seconds * factor**(retry_no - 1)
+          return unfuzzed if fuzz_factor.zero?
+          r = 2 * rand  - 1
+          # r is uniformly distributed in range [-1, 1]
+          unfuzzed * (1 + fuzz_factor * r)
         end
       end
     end
   end
 
   # For backwards compatibility
-  RetryExceptions = Middleware::RetryExceptions
+  module RetryExceptions
+    include Middleware::RetryExceptions
+  end
 end
-

data/lib/qless/middleware/sentry.rb

@@ -1,3 +1,5 @@
+# Encoding: utf-8
+
 require 'raven'
 
 module Qless
@@ -37,7 +39,6 @@ module Qless
           # Qless Web UI.
         end
 
-
         def job_metadata
           {
             jid:      @job.jid,
@@ -55,11 +56,7 @@ module Qless
         def job_history
           @job.queue_history.map do |history_event|
             history_event.each_with_object({}) do |(key, value), hash|
-              hash[key] = if value.is_a?(Time)
-                value.iso8601
-              else
-                value
-              end
+              hash[key] = value.is_a?(Time) ? value.iso8601 : value
             end
           end
         end
@@ -67,4 +64,3 @@ module Qless
     end
   end
 end
-

data/lib/qless/middleware/timeout.rb

@@ -0,0 +1,64 @@
+require 'timeout'
+require 'qless/middleware/requeue_exceptions'
+
+module Qless
+  # Unique error class used when a job is timed out by this middleware.
+  # Allows us to differentiate this timeout from others caused by `::Timeout::Erorr`
+  JobTimedoutError = Class.new(StandardError)
+  InvalidTimeoutError = Class.new(ArgumentError)
+
+  module Middleware
+    # Applies a hard time out. To use this middleware, instantiate it and pass a block; the block
+    # will be passed the job object (which has a `ttl` method for getting the job's remaining TTL),
+    # and the block should return the desired timeout in seconds.
+    # This allows you to set a hard constant time out to a particular job class
+    # (using something like `extend Qless::Middleware::Timeout.new { 60 * 60 }`),
+    # or a variable timeout based on the individual TTLs of each job
+    # (using something like `extend Qless::Middleware::Timeout.new { |job| job.ttl * 1.1 }`).
+    class Timeout < Module
+      def initialize(opts = {})
+        timeout_class = opts.fetch(:timeout_class, ::Timeout)
+        kernel_class = opts.fetch(:kernel_class, Kernel)
+        module_eval do
+          define_method :around_perform do |job|
+            timeout_seconds = yield job
+
+            return super(job) if timeout_seconds.nil?
+
+            if !timeout_seconds.is_a?(Numeric) || timeout_seconds <= 0
+              raise InvalidTimeoutError, "Timeout must be a positive number or nil, " \
+                                         "but was #{timeout_seconds}"
+            end
+
+            begin
+              timeout_class.timeout(timeout_seconds) { super(job) }
+            rescue ::Timeout::Error => e
+              error = JobTimedoutError.new("Qless: job timeout (#{timeout_seconds}) exceeded.")
+              error.set_backtrace(e.backtrace)
+              # The stalled connection to redis might be the cause of the timeout. We cannot rely
+              # on state of connection either (e.g., we might be in the middle of Redis call when
+              # timeout happend). To play it safe, we reconnect.
+              job.reconnect_to_redis
+              job.fail(*Qless.failure_formatter.format(job, error, []))
+              # Since we are leaving with bang (exit!), normal requeue logic does not work.
+              # Do it manually right here.
+              if self.is_a?(::Qless::Middleware::RequeueExceptions) &&
+                 self.requeueable?(JobTimedoutError)
+                self.handle_exception(job, error)
+              end
+
+              # ::Timeout.timeout is dangerous to use as it can leave things in an inconsistent
+              # state. With Redis, for example, we've seen the socket buffer left with unread bytes
+              # on it, which can affect later redis calls. Thus, it's much safer just to exit, and
+              # allow the parent process to restart the worker in a known, clean state.
+              #
+              # We use 73 as a unique exit status for this case. 73 looks
+              # a bit like TE (Timeout::Error)
+              kernel_class.exit!(73)
+            end
+          end
+        end
+      end
+    end
+  end
+end