reqless 0.0.1

data/lib/reqless/lua_script.rb

@@ -0,0 +1,90 @@
+# Encoding: utf-8
+
+require 'digest/sha1'
+
+module Reqless
+  LuaScriptError = Class.new(Reqless::Error)
+
+  # Wraps a lua script. Knows how to reload it if necessary
+  class LuaScript
+    DEFAULT_ON_RELOAD_CALLBACK = proc {}
+    SCRIPT_ROOT = File.expand_path('../lua', __FILE__)
+
+    def initialize(name, redis, options = {})
+      @name  = name
+      @on_reload_callback = options[:on_reload_callback] || DEFAULT_ON_RELOAD_CALLBACK
+      @redis = redis
+      @sha   = Digest::SHA1.hexdigest(script_contents)
+    end
+
+    attr_reader :name, :redis, :sha
+
+    def reload
+      @sha = @redis.script(:load, script_contents)
+      @on_reload_callback.call(@redis, @sha)
+      @sha
+    end
+
+    def call(*argv)
+      handle_no_script_error do
+        _call(*argv)
+      end
+    rescue Redis::CommandError => err
+      if match = err.message.match('user_script:\d+:\s*(\w+.+$)')
+        raise LuaScriptError.new(match[1])
+      else
+        raise err
+      end
+    end
+
+  private
+    def _call(*argv)
+      @redis.evalsha(@sha, keys: [], argv: argv)
+    end
+
+    def handle_no_script_error
+      yield
+    rescue ScriptNotLoadedRedisCommandError
+      reload
+      yield
+    end
+
+    # Module for notifying when a script hasn't yet been loaded
+    module ScriptNotLoadedRedisCommandError
+      MESSAGE = 'NOSCRIPT No matching script. Please use EVAL.'
+
+      def self.===(error)
+        error.is_a?(Redis::CommandError) && error.message.include?(MESSAGE)
+      end
+    end
+
+    def script_contents
+      @script_contents ||= File.read(File.join(SCRIPT_ROOT, "#{@name}.lua"))
+    end
+  end
+
+  # Provides a simple way to load and use lua-based Reqless plugins.
+  # This combines the reqless-lib.lua script plus your custom script
+  # contents all into one script, so that your script can use
+  # Reqless's lua API.
+  class LuaPlugin < LuaScript
+    def initialize(name, redis, plugin_contents)
+      @name  = name
+      @redis = redis
+      @plugin_contents = plugin_contents.gsub(COMMENT_LINES_RE, '')
+      super(name, redis)
+    end
+
+  private
+
+    def script_contents
+      @script_contents ||= [REQLESS_LIB_CONTENTS, @plugin_contents].join("\n\n")
+    end
+
+    COMMENT_LINES_RE = /^\s*--.*$\n?/
+
+    REQLESS_LIB_CONTENTS = File.read(
+      File.join(SCRIPT_ROOT, 'reqless-lib.lua')
+    ).gsub(COMMENT_LINES_RE, '')
+  end
+end

data/lib/reqless/middleware/requeue_exceptions.rb

@@ -0,0 +1,94 @@
+# Encoding: utf-8
+
+module Reqless
+  module Middleware
+    # This middleware is like RetryExceptions, but it doesn't use reqless-core's
+    # internal retry/retry-tracking mechanism. Instead, it re-queues the job
+    # when it fails with a matched error, and increments a counter in the job's
+    # data.
+    #
+    # This is useful for exceptions for which you want a different
+    # backoff/retry strategy. The internal retry mechanism doesn't allow for
+    # separate tracking by exception type, and thus doesn't allow you to retry
+    # different exceptions a different number of times.
+    #
+    # This is particularly useful for handling resource throttling errors,
+    # where you may not want exponential backoff, and you may want the error
+    # to be retried many times, w/o having other transient errors retried so
+    # many times.
+    module RequeueExceptions
+      RequeueableException = Struct.new(:klass, :delay_min, :delay_span, :max_attempts) do
+        def self.from_splat_and_options(*klasses, options)
+          delay_range = options.fetch(:delay_range)
+          delay_min = Float(delay_range.min)
+          delay_span = Float(delay_range.max) - Float(delay_range.min)
+          max_attempts = options.fetch(:max_attempts)
+          klasses.map do |klass|
+            new(klass, delay_min, delay_span, max_attempts)
+          end
+        end
+
+        def delay
+          delay_min + Random.rand(delay_span)
+        end
+
+        def raise_if_exhausted_requeues(error, requeues)
+          raise error if requeues >= max_attempts
+        end
+      end
+
+      def requeue_on(*exceptions, options)
+        RequeueableException.from_splat_and_options(
+          *exceptions, options).each do |exc|
+          requeueable_exceptions[exc.klass] = exc
+        end
+      end
+
+      DEFAULT_ON_REQUEUE_CALLBACK = lambda { |error, job| }
+      def use_on_requeue_callback(&block)
+        @on_requeue_callback = block if block
+      end
+
+      def on_requeue_callback
+        @on_requeue_callback ||= DEFAULT_ON_REQUEUE_CALLBACK
+      end
+
+      def handle_exception(job, error)
+        config = requeuable_exception_for(error)
+
+        requeues_by_exception = (job.data['requeues_by_exception'] ||= {})
+        requeues_by_exception[config.klass.name] ||= 0
+
+        config.raise_if_exhausted_requeues(
+          error, requeues_by_exception[config.klass.name])
+
+        requeues_by_exception[config.klass.name] += 1
+        job.requeue(job.queue_name, delay: config.delay, data: job.data)
+
+        on_requeue_callback.call(error, job)
+      end
+
+      def around_perform(job)
+        super
+      rescue *requeueable_exceptions.keys => e
+        handle_exception(job, e)
+      end
+
+      def requeueable?(exception)
+        requeueable_exceptions.member?(exception)
+      end
+
+      def requeueable_exceptions
+        @requeueable_exceptions ||= {}
+      end
+
+      def requeuable_exception_for(e)
+        requeueable_exceptions.fetch(e.class) do
+          requeueable_exceptions.each do |klass, exc|
+            break exc if klass === e
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reqless/middleware/retry_exceptions.rb

@@ -0,0 +1,72 @@
+# Encoding: utf-8
+
+module Reqless
+  module Middleware
+    # Auto-retries particular errors using reqless-core's internal retry tracking
+    # mechanism. Supports a backoff strategy (typically exponential).
+    #
+    # Note: this does not support varying the number of allowed retries by
+    # exception type. If you want that kind of flexibility, use the
+    # RequeueExceptions middleware instead.
+    module RetryExceptions
+      def around_perform(job)
+        super
+      rescue *retryable_exception_classes => error
+        raise if job.retries_left <= 0
+
+        attempt_num = (job.original_retries - job.retries_left) + 1
+        failure = Reqless.failure_formatter.format(job, error)
+        job.retry(backoff_strategy.call(attempt_num, error), *failure)
+
+        on_retry_callback.call(error, job)
+      end
+
+      def retryable_exception_classes
+        @retryable_exception_classes ||= []
+      end
+
+      def retry_on(*exception_classes)
+        retryable_exception_classes.push(*exception_classes)
+      end
+
+      NO_BACKOFF_STRATEGY = ->(_num, _error) { 0 }
+
+      def use_backoff_strategy(strategy = nil, &block)
+        @backoff_strategy = strategy || block
+      end
+
+      def backoff_strategy
+        @backoff_strategy ||= NO_BACKOFF_STRATEGY
+      end
+
+      DEFAULT_ON_RETRY_CALLBACK = lambda { |error, job| }
+      def use_on_retry_callback(&block)
+        @on_retry_callback = block if block
+      end
+
+      def on_retry_callback
+        @on_retry_callback ||= DEFAULT_ON_RETRY_CALLBACK
+      end
+
+      # If `factor` is omitted it is set to `delay_seconds` to reproduce legacy
+      # behavior.
+      def exponential(delay_seconds, options={})
+        factor = options.fetch(:factor, delay_seconds)
+        fuzz_factor = options.fetch(:fuzz_factor, 0)
+
+        lambda do |retry_no, error|
+          unfuzzed = delay_seconds * factor**(retry_no - 1)
+          return unfuzzed if fuzz_factor.zero?
+          r = 2 * rand  - 1
+          # r is uniformly distributed in range [-1, 1]
+          unfuzzed * (1 + fuzz_factor * r)
+        end
+      end
+    end
+  end
+
+  # For backwards compatibility
+  module RetryExceptions
+    include Middleware::RetryExceptions
+  end
+end

data/lib/reqless/middleware/sentry.rb

@@ -0,0 +1,66 @@
+# Encoding: utf-8
+
+require 'raven'
+
+module Reqless
+  module Middleware
+    # This middleware logs errors to the sentry exception notification service:
+    # http://getsentry.com/
+    module Sentry
+      def around_perform(job)
+        super
+      rescue Exception => e
+        SentryLogger.new(e, job).log
+        raise
+      end
+
+      # Logs a single exception to Sentry, adding pertinent job info.
+      class SentryLogger
+        def initialize(exception, job)
+          @exception, @job = exception, job
+        end
+
+        def log
+          event = ::Raven::Event.capture_exception(@exception) do |evt|
+            evt.extra = { job: job_metadata }
+          end
+
+          safely_send event
+        end
+
+      private
+
+        def safely_send(event)
+          return unless event
+          ::Raven.send(event)
+        rescue
+          # We don't want to silence our errors when the Sentry server
+          # responds with an error. We'll still see the errors on the
+          # Reqless Web UI.
+        end
+
+        def job_metadata
+          {
+            jid:      @job.jid,
+            klass:    @job.klass_name,
+            history:  job_history,
+            data:     @job.data,
+            queue:    @job.queue_name,
+            worker:   @job.worker_name,
+            tags:     @job.tags,
+            priority: @job.priority
+          }
+        end
+
+        # We want to log formatted timestamps rather than integer timestamps
+        def job_history
+          @job.queue_history.map do |history_event|
+            history_event.each_with_object({}) do |(key, value), hash|
+              hash[key] = value.is_a?(Time) ? value.iso8601 : value
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reqless/middleware/timeout.rb

@@ -0,0 +1,63 @@
+require 'timeout'
+require 'reqless/middleware/requeue_exceptions'
+
+module Reqless
+  # Unique error class used when a job is timed out by this middleware.
+  # Allows us to differentiate this timeout from others caused by `::Timeout::Erorr`
+  JobTimedoutError = Class.new(StandardError)
+  InvalidTimeoutError = Class.new(ArgumentError)
+
+  module Middleware
+    # Applies a hard time out. To use this middleware, instantiate it and pass a block; the block
+    # will be passed the job object (which has a `ttl` method for getting the job's remaining TTL),
+    # and the block should return the desired timeout in seconds.
+    # This allows you to set a hard constant time out to a particular job class
+    # (using something like `extend Reqless::Middleware::Timeout.new { 60 * 60 }`),
+    # or a variable timeout based on the individual TTLs of each job
+    # (using something like `extend Reqless::Middleware::Timeout.new { |job| job.ttl * 1.1 }`).
+    class Timeout < Module
+      def initialize(opts = {})
+        timeout_class = opts.fetch(:timeout_class, ::Timeout)
+        kernel_class = opts.fetch(:kernel_class, Kernel)
+        module_eval do
+          define_method :around_perform do |job|
+            timeout_seconds = yield job
+
+            return super(job) if timeout_seconds.nil?
+
+            if !timeout_seconds.is_a?(Numeric) || timeout_seconds <= 0
+              raise InvalidTimeoutError, "Timeout must be a positive number or nil, " \
+                                         "but was #{timeout_seconds}"
+            end
+
+            begin
+              timeout_class.timeout(timeout_seconds) { super(job) }
+            rescue ::Timeout::Error => e
+              error = JobTimedoutError.new(e.message)
+              error.set_backtrace(e.backtrace)
+              # The stalled connection to redis might be the cause of the timeout. We cannot rely
+              # on state of connection either (e.g., we might be in the middle of Redis call when
+              # timeout happend). To play it safe, we reconnect.
+              job.fail(*Reqless.failure_formatter.format(job, error, []))
+              # Since we are leaving with bang (exit!), normal requeue logic does not work.
+              # Do it manually right here.
+              if self.is_a?(::Reqless::Middleware::RequeueExceptions) &&
+                 self.requeueable?(JobTimedoutError)
+                self.handle_exception(job, error)
+              end
+
+              # ::Timeout.timeout is dangerous to use as it can leave things in an inconsistent
+              # state. With Redis, for example, we've seen the socket buffer left with unread bytes
+              # on it, which can affect later redis calls. Thus, it's much safer just to exit, and
+              # allow the parent process to restart the worker in a known, clean state.
+              #
+              # We use 73 as a unique exit status for this case. 73 looks
+              # a bit like TE (Timeout::Error)
+              kernel_class.exit!(73)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/reqless/queue.rb

@@ -0,0 +1,189 @@
+# Encoding: utf-8
+
+require 'reqless/job'
+require 'redis'
+require 'json'
+
+module Reqless
+  # A class for interacting with jobs in different states in a queue. Not meant
+  # to be instantiated directly, it's accessed with Queue#jobs
+  class QueueJobs
+    def initialize(name, client)
+      @name   = name
+      @client = client
+    end
+
+    def running(start = 0, count = 25)
+      JSON.parse(@client.call('queue.jobsByState', 'running', @name, start, count))
+    end
+
+    def throttled(start = 0, count = 25)
+      JSON.parse(@client.call('queue.jobsByState', 'throttled', @name, start, count))
+    end
+
+    def stalled(start = 0, count = 25)
+      JSON.parse(@client.call('queue.jobsByState', 'stalled', @name, start, count))
+    end
+
+    def scheduled(start = 0, count = 25)
+      JSON.parse(@client.call('queue.jobsByState', 'scheduled', @name, start, count))
+    end
+
+    def depends(start = 0, count = 25)
+      JSON.parse(@client.call('queue.jobsByState', 'depends', @name, start, count))
+    end
+
+    def recurring(start = 0, count = 25)
+      JSON.parse(@client.call('queue.jobsByState', 'recurring', @name, start, count))
+    end
+  end
+
+  # A class for interacting with a specific queue. Not meant to be instantiated
+  # directly, it's accessed with Client#queues[...]
+  class Queue
+    attr_reader   :name, :client
+
+    def initialize(name, client)
+      @client = client
+      @name   = name
+    end
+
+    # Our worker name is the same as our client's
+    def worker_name
+      @client.worker_name
+    end
+
+    def jobs
+      @jobs ||= QueueJobs.new(@name, @client)
+    end
+
+    def counts
+      JSON.parse(@client.call('queue.counts', @name))
+    end
+
+    def heartbeat
+      get_config :heartbeat
+    end
+
+    def heartbeat=(value)
+      set_config :heartbeat, value
+    end
+
+    def throttle
+      @throttle ||= Reqless::Throttle.new("ql:q:#{name}", client)
+    end
+
+    def paused?
+      counts['paused']
+    end
+
+    def pause(opts = {})
+      @client.call('queue.pause', name)
+      @client.call('job.timeout', jobs.running(0, -1)) unless opts[:stopjobs].nil?
+    end
+
+    def unpause
+      @client.call('queue.unpause', name)
+    end
+
+    # Put the described job in this queue
+    # Options include:
+    # => priority (int)
+    # => tags (array of strings)
+    # => delay (int)
+    # => throttles (array of strings)
+    def put(klass, data, opts = {})
+      opts = job_options(klass, data, opts)
+      @client.call(
+        'queue.put',
+        worker_name, @name,
+        (opts[:jid] || Reqless.generate_jid),
+        klass.is_a?(String) ? klass : klass.name,
+        *Job.build_opts_array(opts.merge(:data => data)),
+      )
+    end
+
+    # Make a recurring job in this queue
+    # Options include:
+    # => priority (int)
+    # => tags (array of strings)
+    # => retries (int)
+    # => offset (int)
+    def recur(klass, data, interval, opts = {})
+      opts = job_options(klass, data, opts)
+      @client.call(
+        'queue.recurAtInterval',
+        @name,
+        (opts[:jid] || Reqless.generate_jid),
+        klass.is_a?(String) ? klass : klass.name,
+        JSON.generate(data),
+        interval, opts.fetch(:offset, 0),
+        'priority', opts.fetch(:priority, 0),
+        'tags', JSON.generate(opts.fetch(:tags, [])),
+        'retries', opts.fetch(:retries, 5),
+        'backlog', opts.fetch(:backlog, 0)
+      )
+    end
+
+    # Pop a work item off the queue
+    def pop(count = nil)
+      jids = JSON.parse(@client.call('queue.pop', @name, worker_name, (count || 1)))
+      jobs = jids.map { |j| Job.new(@client, j) }
+      count.nil? ? jobs[0] : jobs
+    end
+
+    # Peek at a work item
+    def peek(offset_or_count = nil, count = nil)
+      actual_offset = offset_or_count && count ? offset_or_count : 0
+      actual_count = offset_or_count && count ? count : (offset_or_count || 1)
+      return_single_job = offset_or_count.nil? && count.nil?
+      jids = JSON.parse(@client.call('queue.peek', @name, actual_offset, actual_count))
+      jobs = jids.map { |j| Job.new(@client, j) }
+      return_single_job ? jobs[0] : jobs
+    end
+
+    def stats(date = nil)
+      JSON.parse(@client.call('queue.stats', @name, (date || Time.now.to_f)))
+    end
+
+    # How many items in the queue?
+    def length
+      (@client.redis.multi do |pipeline|
+        pipeline.zcard("ql:q:#{@name}-locks")
+        pipeline.zcard("ql:q:#{@name}-work")
+        pipeline.zcard("ql:q:#{@name}-scheduled")
+      end).inject(0, :+)
+    end
+
+    def to_s
+      "#<Reqless::Queue #{@name}>"
+    end
+    alias_method :inspect, :to_s
+
+    def ==(other)
+      self.class == other.class &&
+      client == other.client &&
+      name.to_s == other.name.to_s
+    end
+    alias eql? ==
+
+    def hash
+      self.class.hash ^ client.hash ^ name.to_s.hash
+    end
+
+  private
+
+    def job_options(klass, data, opts)
+      return opts unless klass.respond_to?(:default_job_options)
+      klass.default_job_options(data).merge(opts)
+    end
+
+    def set_config(config, value)
+      @client.config["#{@name}-#{config}"] = value
+    end
+
+    def get_config(config)
+      @client.config["#{@name}-#{config}"]
+    end
+  end
+end

data/lib/reqless/queue_priority_pattern.rb

@@ -0,0 +1,16 @@
+module Reqless
+  class QueuePriorityPattern
+    attr_reader :pattern, :should_distribute_fairly
+
+    def initialize(pattern, should_distribute_fairly = false)
+      @pattern = pattern
+      @should_distribute_fairly = should_distribute_fairly
+    end
+
+    def ==(other)
+      return self.class == other.class &&
+        self.pattern.join == other.pattern.join &&
+        self.should_distribute_fairly == other.should_distribute_fairly
+    end
+  end
+end