sidekiq-unique-jobs 8.0.3 → 8.0.9

data/lib/sidekiq_unique_jobs/orphans/reaper.rb

@@ -22,11 +22,12 @@ module SidekiqUniqueJobs
       #
       # @return [Hash<Symbol, SidekiqUniqueJobs::Orphans::Reaper] the current implementation of reapers
       REAPERS = {
-        lua: SidekiqUniqueJobs::Orphans::LuaReaper,
+        lua: SidekiqUniqueJobs::Orphans::RubyReaper,
         ruby: SidekiqUniqueJobs::Orphans::RubyReaper,
         none: SidekiqUniqueJobs::Orphans::NullReaper,
         nil => SidekiqUniqueJobs::Orphans::NullReaper,
         false => SidekiqUniqueJobs::Orphans::NullReaper,
+        true => SidekiqUniqueJobs::Orphans::RubyReaper,
       }.freeze
 
       #

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -13,6 +13,9 @@ module SidekiqUniqueJobs
     class RubyReaper < Reaper
       include SidekiqUniqueJobs::Timing
 
+      #
+      # @return [Integer] a best guess of Sidekiq::Launcher::BEAT_PAUSE
+      SIDEKIQ_BEAT_PAUSE = 10
       #
       # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
@@ -74,12 +77,28 @@ module SidekiqUniqueJobs
 
         BatchDelete.call(expired_digests, conn)
         BatchDelete.call(orphans, conn)
+
+        # orphans.each_slice(500) do |chunk|
+        #   conn.pipelined do |pipeline|
+        #     chunk.each do |digest|
+        #       next if belongs_to_job?(digest)
+
+        #       pipeline.zadd(ORPHANED_DIGESTS, now_f, digest)
+        #     end
+        #   end
+        # end
       end
 
       def expired_digests
-        max_score = (start_time - reaper_timeout).to_f
+        conn.zrange(EXPIRING_DIGESTS, 0, max_score, "byscore")
+      end
+
+      def orphaned_digests
+        conn.zrange(ORPHANED_DIGESTS, 0, max_score, "byscore")
+      end
 
-        conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
+      def max_score
+        (start_time - reaper_timeout - SIDEKIQ_BEAT_PAUSE).to_f
       end
 
       #
@@ -89,10 +108,10 @@ module SidekiqUniqueJobs
       # @return [Array<String>] an array of orphaned digests
       #
       def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
-        page = 0
-        per = reaper_count * 2
         orphans = []
-        results = conn.zrange(digests.key, page * per, (page + 1) * per)
+        page    = 0
+        per     = reaper_count * 2
+        results = digests.byscore(0, max_score, offset: page * per, count: (page + 1) * per)
 
         while results.size.positive?
           results.each do |digest|
@@ -107,7 +126,7 @@ module SidekiqUniqueJobs
           break if orphans.size >= reaper_count
 
           page += 1
-          results = conn.zrange(digests.key, page * per, (page + 1) * per)
+          results = digests.byscore(0, max_score, offset: page * per, count: (page + 1) * per)
         end
 
         orphans
@@ -218,7 +237,7 @@ module SidekiqUniqueJobs
       end
 
       def considered_active?(time_f)
-        (Time.now - reaper_timeout).to_f < time_f
+        max_score < time_f
       end
 
       #

data/lib/sidekiq_unique_jobs/redis/sorted_set.rb

@@ -24,10 +24,11 @@ module SidekiqUniqueJobs
       # @return [Hash] when given with_scores: true
       #
       def entries(with_scores: true)
-        entrys = redis { |conn| conn.zrange(key, 0, -1, withscores: with_scores) }
-        return entrys unless with_scores
+        return redis { |conn| conn.zrange(key, 0, -1) } unless with_scores
 
-        entrys.each_with_object({}) { |pair, hash| hash[pair[0]] = pair[1] }
+        redis { |conn| conn.zrange(key, 0, -1, "withscores") }.each_with_object({}) do |pair, hash|
+          hash[pair[0]] = pair[1]
+        end
       end
 
       #
@@ -47,6 +48,14 @@ module SidekiqUniqueJobs
         end
       end
 
+      def byscore(min, max, offset: nil, count: nil)
+        redis do |conn|
+          return conn.zrange(key, min, max, "byscore") unless offset && count
+
+          conn.zrange(key, min, max, "byscore", "limit", offset, count)
+        end
+      end
+
       #
       # Return the zrak of the member
       #

data/lib/sidekiq_unique_jobs/script/caller.rb

@@ -56,7 +56,7 @@ module SidekiqUniqueJobs
       def do_call(file_name, conn, keys, argv)
         argv = argv.dup.push(now_f, debug_lua, max_history, file_name, redis_version)
 
-        Script.execute(file_name, conn, keys: keys, argv: normalize_argv(argv))
+        SidekiqUniqueJobs::Script.execute(file_name, conn, keys: keys, argv: normalize_argv(argv))
       end
 
       #

data/lib/sidekiq_unique_jobs/script/client.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    # Interface to dealing with .lua files
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Client
+      include SidekiqUniqueJobs::Script::Timing
+
+      #
+      # @!attribute [r] logger
+      #   @return [Logger] an instance of a logger
+      attr_reader :logger
+      #
+      # @!attribute [r] file_name
+      #   @return [String] The name of the file to execute
+      attr_reader :config
+      #
+      # @!attribute [r] scripts
+      #   @return [Scripts] the collection with loaded scripts
+      attr_reader :scripts
+
+      def initialize(config)
+        @config  = config
+        @logger  = config.logger
+        @scripts = Scripts.fetch(config.scripts_path)
+      end
+
+      #
+      # Execute a lua script with the provided script_name
+      #
+      # @note this method is recursive if we need to load a lua script
+      #   that wasn't previously loaded.
+      #
+      # @param [Symbol] script_name the name of the script to execute
+      # @param [Redis] conn the redis connection to use for execution
+      # @param [Array<String>] keys script keys
+      # @param [Array<Object>] argv script arguments
+      #
+      # @return value from script
+      #
+      def execute(script_name, conn, keys: [], argv: [])
+        result, elapsed = timed do
+          scripts.execute(script_name, conn, keys: keys, argv: argv)
+        end
+
+        logger.debug("Executed #{script_name}.lua in #{elapsed}ms")
+        result
+      rescue ::RedisClient::CommandError => ex
+        handle_error(script_name, conn, ex) do
+          execute(script_name, conn, keys: keys, argv: argv)
+        end
+      end
+
+      private
+
+      #
+      # Handle errors to allow retrying errors that need retrying
+      #
+      # @param [RedisClient::CommandError] ex exception to handle
+      #
+      # @return [void]
+      #
+      # @yieldreturn [void] yields back to the caller when NOSCRIPT is raised
+      def handle_error(script_name, conn, ex)
+        case ex.message
+        when /NOSCRIPT/
+          handle_noscript(script_name) { return yield }
+        when /BUSY/
+          handle_busy(conn) { return yield }
+        end
+
+        raise unless LuaError.intercepts?(ex)
+
+        script = scripts.fetch(script_name, conn)
+        raise LuaError.new(ex, script)
+      end
+
+      def handle_noscript(script_name)
+        scripts.delete(script_name)
+        yield
+      end
+
+      def handle_busy(conn)
+        scripts.kill(conn)
+      rescue ::RedisClient::CommandError => ex
+        logger.warn(ex)
+      ensure
+        yield
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/config.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    #
+    # Class holding gem configuration
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Config
+      #
+      # @!attribute [r] logger
+      #   @return [Logger] a logger to use for debugging
+      attr_reader :logger
+      #
+      # @!attribute [r] scripts_path
+      #   @return [Pathname] a directory with lua scripts
+      attr_reader :scripts_path
+
+      #
+      # Initialize a new instance of {Config}
+      #
+      #
+      def initialize
+        @conn         = RedisClient.new
+        @logger       = Logger.new($stdout)
+        @scripts_path = nil
+      end
+
+      #
+      # Sets a value for scripts_path
+      #
+      # @param [String, Pathname] obj <description>
+      #
+      # @raise [ArgumentError] when directory does not exist
+      # @raise [ArgumentError] when argument isn't supported
+      #
+      # @return [Pathname]
+      #
+      def scripts_path=(obj)
+        raise ArgumentError, "#{obj} should be a Pathname or String" unless obj.is_a?(Pathname) || obj.is_a?(String)
+        raise ArgumentError, "#{obj} does not exist" unless Dir.exist?(obj.to_s)
+
+        @scripts_path =
+          case obj
+          when String
+            Pathname.new(obj)
+          else
+            obj
+          end
+      end
+
+      #
+      # Sets a value for logger
+      #
+      # @param [Logger] obj a logger to use
+      #
+      # @raise [ArgumentError] when given argument isn't a Logger
+      #
+      # @return [Logger]
+      #
+      def logger=(obj)
+        raise ArgumentError, "#{obj} should be a Logger" unless obj.is_a?(Logger)
+
+        @logger = obj
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/dsl.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    # Interface to dealing with .lua files
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    module DSL
+      MUTEX = Mutex.new
+
+      def self.included(base)
+        base.class_eval do
+          extend ClassMethods
+        end
+      end
+
+      #
+      # Module ClassMethods extends the base class with necessary methods
+      #
+      # @author Mikael Henriksson <mikael@zoolutions.se>
+      #
+      module ClassMethods
+        def execute(file_name, conn, keys: [], argv: [])
+          SidekiqUniqueJobs::Script::Client
+            .new(config)
+            .execute(file_name, conn, keys: keys, argv: argv)
+        end
+
+        # Configure the gem
+        #
+        # This is usually called once at startup of an application
+        # @param [Hash] options global gem options
+        # @option options [String, Pathname] :path
+        # @option options [Logger] :logger (default is Logger.new(STDOUT))
+        # @yield control to the caller when given block
+        def configure(options = {})
+          if block_given?
+            yield config
+          else
+            options.each do |key, val|
+              config.send(:"#{key}=", val)
+            end
+          end
+        end
+
+        #
+        # The current configuration (See: {.configure} on how to configure)
+        #
+        #
+        # @return [Script::Config] the gem configuration
+        #
+        def config
+          MUTEX.synchronize do
+            @config ||= Config.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/logging.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    # Utility module for reducing the number of uses of logger.
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    module Logging
+      def self.included(base)
+        base.send(:extend, self)
+      end
+
+      #
+      # A convenience method for using the configured gem logger
+      #
+      # @see Script#.logger
+      #
+      # @return [Logger]
+      #
+      def logger
+        SidekiqUniqueJobs::Script.logger
+      end
+
+      #
+      # Logs a message at debug level
+      #
+      # @param [String, Exception] message_or_exception the message or exception to log
+      #
+      # @return [void]
+      #
+      # @yield [String, Exception] the message or exception to use for log message
+      #
+      def log_debug(message_or_exception = nil, &block)
+        logger.debug(message_or_exception, &block)
+        nil
+      end
+
+      #
+      # Logs a message at info level
+      #
+      # @param [String, Exception] message_or_exception the message or exception to log
+      #
+      # @return [void]
+      #
+      # @yield [String, Exception] the message or exception to use for log message
+      #
+      def log_info(message_or_exception = nil, &block)
+        logger.info(message_or_exception, &block)
+        nil
+      end
+
+      #
+      # Logs a message at warn level
+      #
+      # @param [String, Exception] message_or_exception the message or exception to log
+      #
+      # @return [void]
+      #
+      # @yield [String, Exception] the message or exception to use for log message
+      #
+      def log_warn(message_or_exception = nil, &block)
+        logger.warn(message_or_exception, &block)
+        nil
+      end
+
+      #
+      # Logs a message at error level
+      #
+      # @param [String, Exception] message_or_exception the message or exception to log
+      #
+      # @return [void]
+      #
+      # @yield [String, Exception] the message or exception to use for log message
+      #
+      def log_error(message_or_exception = nil, &block)
+        logger.error(message_or_exception, &block)
+        nil
+      end
+
+      #
+      # Logs a message at fatal level
+      #
+      # @param [String, Exception] message_or_exception the message or exception to log
+      #
+      # @return [void]
+      #
+      # @yield [String, Exception] the message or exception to use for log message
+      #
+      def log_fatal(message_or_exception = nil, &block)
+        logger.fatal(message_or_exception, &block)
+        nil
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/lua_error.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    #
+    # Misconfiguration is raised when gem is misconfigured
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    #
+    class Misconfiguration < RuntimeError
+    end
+
+    # LuaError raised on errors in Lua scripts
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class LuaError < RuntimeError
+      # Reformats errors raised by redis representing failures while executing
+      # a lua script. The default errors have confusing messages and backtraces,
+      # and a type of +RuntimeError+. This class improves the message and
+      # modifies the backtrace to include the lua script itself in a reasonable
+      # way.
+
+      PATTERN  = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/.freeze
+      LIB_PATH = File.expand_path("..", __dir__).freeze
+      CONTEXT_LINE_NUMBER = 2
+
+      attr_reader :error, :file, :content
+
+      # Is this error one that should be reformatted?
+      #
+      # @param error [StandardError] the original error raised by redis
+      # @return [Boolean] is this an error that should be reformatted?
+      def self.intercepts?(error)
+        PATTERN.match?(error.message)
+      end
+
+      # Initialize a new {LuaError} from an existing redis error, adjusting
+      # the message and backtrace in the process.
+      #
+      # @param error [StandardError] the original error raised by redis
+      # @param script [Script] a DTO with information about the script
+      #
+      def initialize(error, script)
+        @error        = error
+        @file         = script.path
+        @content      = script.source
+        @backtrace    = @error.backtrace
+
+        @error.message.match(PATTERN) do |regexp_match|
+          line_number   = regexp_match[2].to_i
+          message       = regexp_match[3]
+          error_context = generate_error_context(content, line_number)
+
+          super("#{message}\n\n#{error_context}\n\n")
+          set_backtrace(generate_backtrace(file, line_number))
+        end
+      end
+
+      private
+
+      # :nocov:
+      def generate_error_context(content, line_number)
+        lines                 = content.lines.to_a
+        beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
+        ending_line_number    = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
+        line_number_width     = ending_line_number.to_s.length
+
+        (beginning_line_number..ending_line_number).map do |number|
+          indicator = (number == line_number) ? "=>" : "  "
+          formatted_number = format("%#{line_number_width}d", number)
+          " #{indicator} #{formatted_number}: #{lines[number - 1]}"
+        end.join.chomp
+      end
+
+      # :nocov:
+      def generate_backtrace(file, line_number)
+        pre_gem                 = backtrace_before_entering_gem(@backtrace)
+        index_of_first_gem_line = (@backtrace.size - pre_gem.size - 1)
+
+        pre_gem.unshift(@backtrace[index_of_first_gem_line])
+        pre_gem.unshift("#{file}:#{line_number}")
+        pre_gem
+      end
+
+      # :nocov:
+      def backtrace_before_entering_gem(backtrace)
+        backtrace.reverse.take_while { |line| !line_from_gem(line) }.reverse
+      end
+
+      # :nocov:
+      def line_from_gem(line)
+        line.split(":").first.include?(LIB_PATH)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/script.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    # Interface to dealing with .lua files
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Script
+      def self.load(name, root_path, conn)
+        script = new(name: name, root_path: root_path)
+        script.load(conn)
+      end
+
+      #
+      # @!attribute [r] script_name
+      #   @return [Symbol, String] the name of the script without extension
+      attr_reader :name
+      #
+      # @!attribute [r] script_path
+      #   @return [String] the path to the script on disk
+      attr_reader :path
+      #
+      # @!attribute [r] root_path
+      #   @return [Pathname]
+      attr_reader :root_path
+      #
+      # @!attribute [r] source
+      #   @return [String] the source code of the lua script
+      attr_reader :source
+      #
+      # @!attribute [rw] sha
+      #   @return [String] the sha of the script
+      attr_reader :sha
+      #
+      # @!attribute [rw] call_count
+      #   @return [Integer] the number of times the script was called/executed
+      attr_reader :call_count
+
+      def initialize(name:, root_path:)
+        @name      = name
+        @root_path = root_path
+        @path      = root_path.join("#{name}.lua").to_s
+        @source    = render_file
+        @sha       = compiled_sha
+        @call_count = 0
+      end
+
+      def ==(other)
+        sha == compiled_sha && compiled_sha == other.sha
+      end
+
+      def increment_call_count
+        @call_count += 1
+      end
+
+      def changed?
+        compiled_sha != sha
+      end
+
+      def render_file
+        Template.new(root_path).render(path)
+      end
+
+      def compiled_sha
+        Digest::SHA1.hexdigest(source)
+      end
+
+      def load(conn)
+        @sha = conn.script(:load, source)
+
+        self
+      end
+    end
+  end
+end