sidekiq-unique-jobs 8.0.3 → 8.0.10

data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua

@@ -1,19 +1,29 @@
 local function delete_from_sorted_set(name, digest)
-  local per   = 50
-  local total = redis.call("zcard", name)
-  local index = 0
-  local result
+  local score  = redis.call("ZSCORE", "uniquejobs:digests", digest)
+  local total  = redis.call("ZCARD", name)
+  local per    = 50
+
+  for offset = 0, total, per do
+    local items
+
+    if score then
+      items = redis.call("ZRANGE", name, score, "+inf", "BYSCORE", "LIMIT", offset, per)
+    else
+      items = redis.call("ZRANGE", name, offset, offset + per -1)
+    end
+
+    if #items == 0 then
+      break
+    end
 
-  while (index < total) do
-    local items = redis.call("ZRANGE", name, index, index + per -1)
     for _, item in pairs(items) do
       if string.find(item, digest) then
         redis.call("ZREM", name, item)
-        result = item
-        break
+
+        return item
       end
     end
-    index = index + per
   end
-  return result
+
+  return nil
 end

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -10,19 +10,20 @@ local digests   = KEYS[7]
 
 
 -------- BEGIN lock arguments ---------
-local job_id    = ARGV[1]
-local pttl      = tonumber(ARGV[2])
-local lock_type = ARGV[3]
-local limit     = tonumber(ARGV[4])
+local job_id     = ARGV[1]
+local pttl       = tonumber(ARGV[2])
+local lock_type  = ARGV[3]
+local limit      = tonumber(ARGV[4])
+local lock_score = ARGV[5]
 -------- END lock arguments -----------
 
 
 --------  BEGIN injected arguments --------
-local current_time = tonumber(ARGV[5])
-local debug_lua    = tostring(ARGV[6]) == "1"
-local max_history  = tonumber(ARGV[7])
-local script_name  = tostring(ARGV[8]) .. ".lua"
-local redisversion = ARGV[9]
+local current_time = tonumber(ARGV[6])
+local debug_lua    = tostring(ARGV[7]) == "1"
+local max_history  = tonumber(ARGV[8])
+local script_name  = tostring(ARGV[9]) .. ".lua"
+local redisversion = ARGV[10]
 ---------  END injected arguments ---------
 
 
@@ -75,21 +76,28 @@ if lock_type ~= "until_expired" then
   redis.call("HDEL", locked, job_id)
 end
 
+if redis.call("LLEN", primed) == 0 then
+  log_debug("UNLINK", primed)
+  redis.call("UNLINK", primed)
+end
+
 local locked_count = redis.call("HLEN", locked)
 
-if locked_count and locked_count < 1 then
+if locked_count < 1 then
   log_debug("UNLINK", locked)
   redis.call("UNLINK", locked)
 end
 
-if redis.call("LLEN", primed) == 0 then
-  log_debug("UNLINK", primed)
-  redis.call("UNLINK", primed)
-end
-
-if limit and limit <= 1 and locked_count and locked_count <= 1 then
-  log_debug("ZREM", digests, digest)
-  redis.call("ZREM", digests, digest)
+if limit then
+  if limit <= 1 and locked_count <= 1 then
+    log_debug("ZREM", digests, digest)
+    redis.call("ZREM", digests, digest)
+  end
+else
+  if locked_count <= 1 then
+    log_debug("ZREM", digests, digest)
+    redis.call("ZREM", digests, digest)
+  end
 end
 
 log_debug("LPUSH", queued, "1")

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Client
-      include Sidekiq::ClientMiddleware if defined?(Sidekiq::ClientMiddleware)
+      include Sidekiq::ClientMiddleware
 
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Server
-      include Sidekiq::ServerMiddleware if defined?(Sidekiq::ServerMiddleware)
+      include Sidekiq::ServerMiddleware
 
       # prepend "SidekiqUniqueJobs::Middleware"
       # @!parse prepends SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -19,15 +19,15 @@ module SidekiqUniqueJobs
     # This method runs before (prepended) the actual middleware implementation.
     #   This is done to reduce duplication
     #
-    # @param [Sidekiq::Worker] worker_class
+    # @param [Sidekiq::Job] worker_class
     # @param [Hash] item a sidekiq job hash
     # @param [String] queue name of the queue
     # @param [ConnectionPool] redis_pool only used for compatility reasons
     #
-    # @return [yield<super>] <description>
+    # @return [yield<super>] call the rest of the middleware stack
     #
-    # @yieldparam [<type>] if <description>
-    # @yieldreturn [<type>] <describe what yield should return>
+    # @yieldparam [void] if uniquejobs is disable
+    # @yieldreturn [void] delegate back to other sidekiq middleware
     def call(worker_class, item, queue, redis_pool = nil)
       @item       = item
       @queue      = queue
@@ -35,7 +35,7 @@ module SidekiqUniqueJobs
       self.job_class = worker_class
       return yield if unique_disabled?
 
-      SidekiqUniqueJobs::Job.prepare(item) unless item[LOCK_DIGEST]
+      SidekiqUniqueJobs::Job.prepare(item)
 
       with_logging_context do
         super

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -65,7 +65,7 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of keys deleted
       #
       def delete_lock
-        digests.delete_by_digest(lock_digest)
+        digests.delete_by_digest(lock_digest, runtime: false)
       end
 
       #

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
       #   This will mess up sidekiq stats because a new job is created
       def call
         if sidekiq_job_class?
-          if job_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
+          if job_class.set(queue: item["queue"].to_sym).perform_in(schedule_in, *item[ARGS])
             reflect(:rescheduled, item)
           else
             reflect(:reschedule_failed, item)
@@ -30,6 +30,10 @@ module SidekiqUniqueJobs
           reflect(:unknown_sidekiq_worker, item)
         end
       end
+
+      def schedule_in
+        job_class.get_sidekiq_options["schedule_in"] || 5
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -39,12 +39,13 @@ module SidekiqUniqueJobs
         self.task = test_task || default_task
 
         with_logging_context do
-          register_reaper_process
-          log_info("Starting Reaper")
+          if register_reaper_process
+            log_info("Starting Reaper")
 
-          task.add_observer(Observer.new)
-          task.execute
-          task
+            task.add_observer(Observer.new)
+            task.execute
+            task
+          end
         end
       end
 
@@ -192,7 +193,7 @@ module SidekiqUniqueJobs
       # @return [void]
       #
       def register_reaper_process
-        redis { |conn| conn.set(UNIQUE_REAPER, current_timestamp, nx: true, ex: drift_reaper_interval) }
+        redis { |conn| conn.set(UNIQUE_REAPER, current_timestamp, "nx", "ex", drift_reaper_interval) }
       end
 
       #
@@ -202,7 +203,7 @@ module SidekiqUniqueJobs
       # @return [void]
       #
       def refresh_reaper_mutex
-        redis { |conn| conn.set(UNIQUE_REAPER, current_timestamp, ex: drift_reaper_interval) }
+        redis { |conn| conn.set(UNIQUE_REAPER, current_timestamp, "ex", drift_reaper_interval) }
       end
 
       #

data/lib/sidekiq_unique_jobs/orphans/reaper.rb

@@ -22,11 +22,12 @@ module SidekiqUniqueJobs
       #
       # @return [Hash<Symbol, SidekiqUniqueJobs::Orphans::Reaper] the current implementation of reapers
       REAPERS = {
-        lua: SidekiqUniqueJobs::Orphans::LuaReaper,
+        lua: SidekiqUniqueJobs::Orphans::RubyReaper,
         ruby: SidekiqUniqueJobs::Orphans::RubyReaper,
         none: SidekiqUniqueJobs::Orphans::NullReaper,
         nil => SidekiqUniqueJobs::Orphans::NullReaper,
         false => SidekiqUniqueJobs::Orphans::NullReaper,
+        true => SidekiqUniqueJobs::Orphans::RubyReaper,
       }.freeze
 
       #

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -13,6 +13,9 @@ module SidekiqUniqueJobs
     class RubyReaper < Reaper
       include SidekiqUniqueJobs::Timing
 
+      #
+      # @return [Integer] a best guess of Sidekiq::Launcher::BEAT_PAUSE
+      SIDEKIQ_BEAT_PAUSE = 10
       #
       # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
@@ -74,12 +77,28 @@ module SidekiqUniqueJobs
 
         BatchDelete.call(expired_digests, conn)
         BatchDelete.call(orphans, conn)
+
+        # orphans.each_slice(500) do |chunk|
+        #   conn.pipelined do |pipeline|
+        #     chunk.each do |digest|
+        #       next if belongs_to_job?(digest)
+
+        #       pipeline.zadd(ORPHANED_DIGESTS, now_f, digest)
+        #     end
+        #   end
+        # end
       end
 
       def expired_digests
-        max_score = (start_time - reaper_timeout).to_f
+        conn.zrange(EXPIRING_DIGESTS, 0, max_score, "byscore")
+      end
+
+      def orphaned_digests
+        conn.zrange(ORPHANED_DIGESTS, 0, max_score, "byscore")
+      end
 
-        conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
+      def max_score
+        (start_time - reaper_timeout - SIDEKIQ_BEAT_PAUSE).to_f
       end
 
       #
@@ -89,10 +108,10 @@ module SidekiqUniqueJobs
       # @return [Array<String>] an array of orphaned digests
       #
       def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
-        page = 0
-        per = reaper_count * 2
         orphans = []
-        results = conn.zrange(digests.key, page * per, (page + 1) * per)
+        page    = 0
+        per     = reaper_count * 2
+        results = digests.byscore(0, max_score, offset: page * per, count: (page + 1) * per)
 
         while results.size.positive?
           results.each do |digest|
@@ -107,7 +126,7 @@ module SidekiqUniqueJobs
           break if orphans.size >= reaper_count
 
           page += 1
-          results = conn.zrange(digests.key, page * per, (page + 1) * per)
+          results = digests.byscore(0, max_score, offset: page * per, count: (page + 1) * per)
         end
 
         orphans
@@ -218,7 +237,7 @@ module SidekiqUniqueJobs
       end
 
       def considered_active?(time_f)
-        (Time.now - reaper_timeout).to_f < time_f
+        max_score < time_f
       end
 
       #

data/lib/sidekiq_unique_jobs/redis/sorted_set.rb

@@ -24,10 +24,11 @@ module SidekiqUniqueJobs
       # @return [Hash] when given with_scores: true
       #
       def entries(with_scores: true)
-        entrys = redis { |conn| conn.zrange(key, 0, -1, withscores: with_scores) }
-        return entrys unless with_scores
+        return redis { |conn| conn.zrange(key, 0, -1) } unless with_scores
 
-        entrys.each_with_object({}) { |pair, hash| hash[pair[0]] = pair[1] }
+        redis { |conn| conn.zrange(key, 0, -1, "withscores") }.each_with_object({}) do |pair, hash|
+          hash[pair[0]] = pair[1]
+        end
       end
 
       #
@@ -47,6 +48,14 @@ module SidekiqUniqueJobs
         end
       end
 
+      def byscore(min, max, offset: nil, count: nil)
+        redis do |conn|
+          return conn.zrange(key, min, max, "byscore") unless offset && count
+
+          conn.zrange(key, min, max, "byscore", "limit", offset, count)
+        end
+      end
+
       #
       # Return the zrak of the member
       #

data/lib/sidekiq_unique_jobs/script/caller.rb

@@ -56,7 +56,7 @@ module SidekiqUniqueJobs
       def do_call(file_name, conn, keys, argv)
         argv = argv.dup.push(now_f, debug_lua, max_history, file_name, redis_version)
 
-        Script.execute(file_name, conn, keys: keys, argv: normalize_argv(argv))
+        SidekiqUniqueJobs::Script.execute(file_name, conn, keys: keys, argv: normalize_argv(argv))
       end
 
       #

data/lib/sidekiq_unique_jobs/script/client.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    # Interface to dealing with .lua files
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Client
+      include SidekiqUniqueJobs::Script::Timing
+
+      #
+      # @!attribute [r] logger
+      #   @return [Logger] an instance of a logger
+      attr_reader :logger
+      #
+      # @!attribute [r] file_name
+      #   @return [String] The name of the file to execute
+      attr_reader :config
+      #
+      # @!attribute [r] scripts
+      #   @return [Scripts] the collection with loaded scripts
+      attr_reader :scripts
+
+      def initialize(config)
+        @config  = config
+        @logger  = config.logger
+        @scripts = Scripts.fetch(config.scripts_path)
+      end
+
+      #
+      # Execute a lua script with the provided script_name
+      #
+      # @note this method is recursive if we need to load a lua script
+      #   that wasn't previously loaded.
+      #
+      # @param [Symbol] script_name the name of the script to execute
+      # @param [Redis] conn the redis connection to use for execution
+      # @param [Array<String>] keys script keys
+      # @param [Array<Object>] argv script arguments
+      #
+      # @return value from script
+      #
+      def execute(script_name, conn, keys: [], argv: [])
+        result, elapsed = timed do
+          scripts.execute(script_name, conn, keys: keys, argv: argv)
+        end
+
+        logger.debug("Executed #{script_name}.lua in #{elapsed}ms")
+        result
+      rescue ::RedisClient::CommandError => ex
+        handle_error(script_name, conn, ex) do
+          execute(script_name, conn, keys: keys, argv: argv)
+        end
+      end
+
+      private
+
+      #
+      # Handle errors to allow retrying errors that need retrying
+      #
+      # @param [RedisClient::CommandError] ex exception to handle
+      #
+      # @return [void]
+      #
+      # @yieldreturn [void] yields back to the caller when NOSCRIPT is raised
+      def handle_error(script_name, conn, ex)
+        case ex.message
+        when /NOSCRIPT/
+          handle_noscript(script_name) { return yield }
+        when /BUSY/
+          handle_busy(conn) { return yield }
+        end
+
+        raise unless LuaError.intercepts?(ex)
+
+        script = scripts.fetch(script_name, conn)
+        raise LuaError.new(ex, script)
+      end
+
+      def handle_noscript(script_name)
+        scripts.delete(script_name)
+        yield
+      end
+
+      def handle_busy(conn)
+        scripts.kill(conn)
+      rescue ::RedisClient::CommandError => ex
+        logger.warn(ex)
+      ensure
+        yield
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/config.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    #
+    # Class holding gem configuration
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Config
+      #
+      # @!attribute [r] logger
+      #   @return [Logger] a logger to use for debugging
+      attr_reader :logger
+      #
+      # @!attribute [r] scripts_path
+      #   @return [Pathname] a directory with lua scripts
+      attr_reader :scripts_path
+
+      #
+      # Initialize a new instance of {Config}
+      #
+      #
+      def initialize
+        @conn         = RedisClient.new
+        @logger       = Logger.new($stdout)
+        @scripts_path = nil
+      end
+
+      #
+      # Sets a value for scripts_path
+      #
+      # @param [String, Pathname] obj <description>
+      #
+      # @raise [ArgumentError] when directory does not exist
+      # @raise [ArgumentError] when argument isn't supported
+      #
+      # @return [Pathname]
+      #
+      def scripts_path=(obj)
+        raise ArgumentError, "#{obj} should be a Pathname or String" unless obj.is_a?(Pathname) || obj.is_a?(String)
+        raise ArgumentError, "#{obj} does not exist" unless Dir.exist?(obj.to_s)
+
+        @scripts_path =
+          case obj
+          when String
+            Pathname.new(obj)
+          else
+            obj
+          end
+      end
+
+      #
+      # Sets a value for logger
+      #
+      # @param [Logger] obj a logger to use
+      #
+      # @raise [ArgumentError] when given argument isn't a Logger
+      #
+      # @return [Logger]
+      #
+      def logger=(obj)
+        raise ArgumentError, "#{obj} should be a Logger" unless obj.is_a?(Logger)
+
+        @logger = obj
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/dsl.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    # Interface to dealing with .lua files
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    module DSL
+      MUTEX = Mutex.new
+
+      def self.included(base)
+        base.class_eval do
+          extend ClassMethods
+        end
+      end
+
+      #
+      # Module ClassMethods extends the base class with necessary methods
+      #
+      # @author Mikael Henriksson <mikael@zoolutions.se>
+      #
+      module ClassMethods
+        def execute(file_name, conn, keys: [], argv: [])
+          SidekiqUniqueJobs::Script::Client
+            .new(config)
+            .execute(file_name, conn, keys: keys, argv: argv)
+        end
+
+        # Configure the gem
+        #
+        # This is usually called once at startup of an application
+        # @param [Hash] options global gem options
+        # @option options [String, Pathname] :path
+        # @option options [Logger] :logger (default is Logger.new(STDOUT))
+        # @yield control to the caller when given block
+        def configure(options = {})
+          if block_given?
+            yield config
+          else
+            options.each do |key, val|
+              config.send(:"#{key}=", val)
+            end
+          end
+        end
+
+        #
+        # The current configuration (See: {.configure} on how to configure)
+        #
+        #
+        # @return [Script::Config] the gem configuration
+        #
+        def config
+          MUTEX.synchronize do
+            @config ||= Config.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/logging.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    # Utility module for reducing the number of uses of logger.
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    module Logging
+      def self.included(base)
+        base.send(:extend, self)
+      end
+
+      #
+      # A convenience method for using the configured gem logger
+      #
+      # @see Script#.logger
+      #
+      # @return [Logger]
+      #
+      def logger
+        SidekiqUniqueJobs::Script.logger
+      end
+
+      #
+      # Logs a message at debug level
+      #
+      # @param [String, Exception] message_or_exception the message or exception to log
+      #
+      # @return [void]
+      #
+      # @yield [String, Exception] the message or exception to use for log message
+      #
+      def log_debug(message_or_exception = nil, &block)
+        logger.debug(message_or_exception, &block)
+        nil
+      end
+
+      #
+      # Logs a message at info level
+      #
+      # @param [String, Exception] message_or_exception the message or exception to log
+      #
+      # @return [void]
+      #
+      # @yield [String, Exception] the message or exception to use for log message
+      #
+      def log_info(message_or_exception = nil, &block)
+        logger.info(message_or_exception, &block)
+        nil
+      end
+
+      #
+      # Logs a message at warn level
+      #
+      # @param [String, Exception] message_or_exception the message or exception to log
+      #
+      # @return [void]
+      #
+      # @yield [String, Exception] the message or exception to use for log message
+      #
+      def log_warn(message_or_exception = nil, &block)
+        logger.warn(message_or_exception, &block)
+        nil
+      end
+
+      #
+      # Logs a message at error level
+      #
+      # @param [String, Exception] message_or_exception the message or exception to log
+      #
+      # @return [void]
+      #
+      # @yield [String, Exception] the message or exception to use for log message
+      #
+      def log_error(message_or_exception = nil, &block)
+        logger.error(message_or_exception, &block)
+        nil
+      end
+
+      #
+      # Logs a message at fatal level
+      #
+      # @param [String, Exception] message_or_exception the message or exception to log
+      #
+      # @return [void]
+      #
+      # @yield [String, Exception] the message or exception to use for log message
+      #
+      def log_fatal(message_or_exception = nil, &block)
+        logger.fatal(message_or_exception, &block)
+        nil
+      end
+    end
+  end
+end