sidekiq-unique-jobs 7.0.0.beta26 → 7.0.1

data/lib/sidekiq-unique-jobs.rb

@@ -1,5 +1,3 @@
 # frozen_string_literal: true
 
 require "sidekiq_unique_jobs"
-
-SidekiqUniqueJobs::Middleware.configure

data/lib/sidekiq_unique_jobs.rb

@@ -71,3 +71,4 @@ require "sidekiq_unique_jobs/config"
 require "sidekiq_unique_jobs/sidekiq_unique_jobs"
 require "sidekiq_unique_jobs/update_version"
 require "sidekiq_unique_jobs/upgrade_locks"
+require "sidekiq_unique_jobs/server"

data/lib/sidekiq_unique_jobs/changelog.rb

@@ -7,6 +7,13 @@ module SidekiqUniqueJobs
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   class Changelog < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
+    DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
+    SCAN_PATTERN  = "*"
+
     def initialize
       super(CHANGELOGS)
     end
@@ -34,10 +41,10 @@ module SidekiqUniqueJobs
     #
     # @return [Array<Hash>] an array of entries
     #
-    def entries(pattern: "*", count: nil)
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
       options = {}
-      options[:match] = pattern if pattern
-      options[:count] = count if count
+      options[:match] = pattern
+      options[:count] = count
 
       redis do |conn|
         conn.zscan_each(key, **options).to_a.map { |entry| load_json(entry[0]) }
@@ -53,7 +60,7 @@ module SidekiqUniqueJobs
     #
     # @return [Array<Integer, Integer, Array<Hash>] the total size, next cursor and changelog entries
     #
-    def page(cursor, pattern: "*", page_size: 100)
+    def page(cursor: 0, pattern: "*", page_size: 100)
       redis do |conn|
         total_size, result = conn.multi do
           conn.zcard(key)

data/lib/sidekiq_unique_jobs/config.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
                                                    :lock_timeout,
                                                    :lock_ttl,
                                                    :enabled,
-                                                   :unique_prefix,
+                                                   :lock_prefix,
                                                    :logger,
                                                    :locks,
                                                    :strategies,
@@ -127,7 +127,7 @@ module SidekiqUniqueJobs
     #   default_lock_timeout: 0,
     #   default_lock_ttl: nil,
     #   enabled: true,
-    #   unique_prefix: "uniquejobs",
+    #   lock_prefix: "uniquejobs",
     #   logger: #<Sidekiq::Logger:0x00007f81e096b0e0 @level=1 ...>,
     #   locks: {
     #     around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,

data/lib/sidekiq_unique_jobs/constants.rb

@@ -11,6 +11,7 @@ module SidekiqUniqueJobs
   AT                    ||= "at"
   CHANGELOGS            ||= "uniquejobs:changelog"
   CLASS                 ||= "class"
+  CREATED_AT            ||= "created_at"
   DEAD_VERSION          ||= "uniquejobs:dead"
   DIGESTS               ||= "uniquejobs:digests"
   ERRORS                ||= "errors"
@@ -19,6 +20,7 @@ module SidekiqUniqueJobs
   LIVE_VERSION          ||= "uniquejobs:live"
   LOCK                  ||= "lock"
   LOCK_ARGS             ||= "lock_args"
+  LOCK_ARGS_METHOD      ||= "lock_args_method"
   LOCK_DIGEST           ||= "lock_digest"
   LOCK_EXPIRATION       ||= "lock_expiration"
   LOCK_INFO             ||= "lock_info"
@@ -31,6 +33,7 @@ module SidekiqUniqueJobs
   ON_CLIENT_CONFLICT    ||= "on_client_conflict"
   ON_CONFLICT           ||= "on_conflict"
   ON_SERVER_CONFLICT    ||= "on_server_conflict"
+  PAYLOAD               ||= "payload"
   PROCESSES             ||= "processes"
   QUEUE                 ||= "queue"
   RETRY                 ||= "retry"
@@ -43,6 +46,7 @@ module SidekiqUniqueJobs
   UNIQUE_ACROSS_QUEUES  ||= "unique_across_queues"
   UNIQUE_ACROSS_WORKERS ||= "unique_across_workers"
   UNIQUE_ARGS           ||= "unique_args"
+  UNIQUE_ARGS_METHOD    ||= "unique_args_method"
   UNIQUE_DIGEST         ||= "unique_digest"
   UNIQUE_PREFIX         ||= "unique_prefix"
   UNIQUE_REAPER         ||= "uniquejobs:reaper"

data/lib/sidekiq_unique_jobs/digests.rb

@@ -78,7 +78,7 @@ module SidekiqUniqueJobs
     def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
       options = {}
       options[:match] = pattern
-      options[:count] = count if count
+      options[:count] = count
 
       result = redis { |conn| conn.zscan_each(key, **options).to_a }
 

data/lib/sidekiq_unique_jobs/job.rb

@@ -52,7 +52,7 @@ module SidekiqUniqueJobs
     end
 
     def add_lock_prefix(item)
-      item[LOCK_PREFIX] ||= SidekiqUniqueJobs.config.unique_prefix
+      item[LOCK_PREFIX] ||= SidekiqUniqueJobs.config.lock_prefix
     end
   end
 end

data/lib/sidekiq_unique_jobs/lock/validator.rb

@@ -12,7 +12,8 @@ module SidekiqUniqueJobs
       # @return [Hash] a hash mapping of deprecated keys and their new value
       DEPRECATED_KEYS = {
         UNIQUE.to_sym => LOCK.to_sym,
-        UNIQUE_ARGS.to_sym => LOCK_ARGS.to_sym,
+        UNIQUE_ARGS.to_sym => LOCK_ARGS_METHOD.to_sym,
+        LOCK_ARGS.to_sym => LOCK_ARGS_METHOD.to_sym,
         UNIQUE_PREFIX.to_sym => LOCK_PREFIX.to_sym,
       }.freeze
 

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -43,7 +43,7 @@ module SidekiqUniqueJobs
           end
         end
       ensure
-        locksmith.unlock
+        locksmith.unlock!
       end
 
       private

data/lib/sidekiq_unique_jobs/lock_args.rb

@@ -97,8 +97,8 @@ module SidekiqUniqueJobs
     end
 
     # The method to use for filtering unique arguments
-    def lock_args_method # rubocop:disable Metrics/CyclomaticComplexity
-      @lock_args_method ||= worker_options[LOCK_ARGS] || worker_options[UNIQUE_ARGS]
+    def lock_args_method
+      @lock_args_method ||= worker_options.slice(LOCK_ARGS_METHOD, UNIQUE_ARGS_METHOD).values.first
       @lock_args_method ||= :lock_args if worker_method_defined?(:lock_args)
       @lock_args_method ||= :unique_args if worker_method_defined?(:unique_args)
       @lock_args_method ||= default_lock_args_method
@@ -106,8 +106,8 @@ module SidekiqUniqueJobs
 
     # The global worker options defined in Sidekiq directly
     def default_lock_args_method
-      default_worker_options[LOCK_ARGS] ||
-        default_worker_options[UNIQUE_ARGS]
+      default_worker_options[LOCK_ARGS_METHOD] ||
+        default_worker_options[UNIQUE_ARGS_METHOD]
     end
 
     #

data/lib/sidekiq_unique_jobs/lock_config.rb

@@ -98,6 +98,8 @@ module SidekiqUniqueJobs
     # @return [String]
     #
     def errors_as_string
+      return if valid?
+
       @errors_as_string ||= begin
         error_msg = +"\t"
         error_msg << errors.map { |key, val| "#{key}: :#{val}" }.join("\n\t")

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -87,7 +87,7 @@ module SidekiqUniqueJobs
     #
     def lock(&block)
       redis(redis_pool) do |conn|
-        return lock_async(conn, &block) if block_given?
+        return lock_async(conn, &block) if block
 
         lock_sync(conn) do
           return job_id

data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua

@@ -25,7 +25,7 @@ local redisversion = tostring(ARGV[5])
 --------  BEGIN delete_by_digest.lua --------
 local counter       = 0
 local redis_version = toversion(redisversion)
-local del_cmd   = "DEL"
+local del_cmd       = "DEL"
 
 log_debug("BEGIN delete_by_digest:", digest)
 

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -69,8 +69,7 @@ redis.call("LREM", queued, -1, job_id)
 log_debug("LREM", primed, 1, job_id)
 redis.call("LREM", primed, 1, job_id)
 
--- The Sidekiq client should only set pttl for until_expired
--- The Sidekiq server should set pttl for all other jobs
+-- The Sidekiq client sets pttl
 if pttl and pttl > 0 then
   log_debug("PEXPIRE", digest, pttl)
   redis.call("PEXPIRE", digest, pttl)

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -7,15 +7,16 @@ local retry_set    = KEYS[3]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
-local reaper_count  = tonumber(ARGV[1])
+local reaper_count = tonumber(ARGV[1])
+local threshold    = tonumber(ARGV[2])
 --------  END argv  ---------
 
 --------  BEGIN injected arguments --------
-local current_time = tonumber(ARGV[2])
-local debug_lua    = ARGV[3] == "true"
-local max_history  = tonumber(ARGV[4])
-local script_name  = ARGV[5] .. ".lua"
-local redisversion = ARGV[6]
+local current_time = tonumber(ARGV[3])
+local debug_lua    = ARGV[4] == "true"
+local max_history  = tonumber(ARGV[5])
+local script_name  = ARGV[6] .. ".lua"
+local redisversion = ARGV[7]
 ---------  END injected arguments ---------
 
 
@@ -65,7 +66,7 @@ repeat
     -- TODO: Add check for jobs checked out by process
     if found ~= true then
       log_debug("Searching for digest:", digest, "in process sets")
-      found = find_digest_in_process_set(digest)
+      found = find_digest_in_process_set(digest, threshold)
     end
 
     if found ~= true then

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua

@@ -1,4 +1,4 @@
-local function find_digest_in_process_set(digest)
+local function find_digest_in_process_set(digest, threshold)
   local process_cursor = 0
   local job_cursor     = 0
   local pattern        = "*" .. digest .. "*"
@@ -26,11 +26,18 @@ local function find_digest_in_process_set(digest)
         log_debug("No entries in:", workers_key)
       else
         for i = 1, #jobs, 2 do
-          if string.find(jobs[i +1], digest) then
+          local jobstr = jobs[i +1]
+          if string.find(jobstr, digest) then
             log_debug("Found digest", digest, "in:", workers_key)
             found = true
             break
           end
+
+          local job = cjson.decode(jobstr)
+          if job.created_at > threshold then
+            found = true
+            break
+          end
         end
       end
 

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -11,63 +11,6 @@ module SidekiqUniqueJobs
     include SidekiqUniqueJobs::OptionsWithFallback
     include SidekiqUniqueJobs::JSON
 
-    #
-    # Configure both server and client
-    #
-    def self.configure
-      configure_server
-      configure_client
-    end
-
-    #
-    # Configures the Sidekiq server
-    #
-    def self.configure_server # rubocop:disable Metrics/MethodLength
-      Sidekiq.configure_server do |config|
-        config.client_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Client
-          end
-        end
-
-        config.server_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Server)
-            chain.insert_after Apartment::Sidekiq::Middleware::Server, SidekiqUniqueJobs::Middleware::Server
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Server
-          end
-        end
-
-        config.on(:startup) do
-          SidekiqUniqueJobs::UpdateVersion.call
-          SidekiqUniqueJobs::UpgradeLocks.call
-
-          SidekiqUniqueJobs::Orphans::Manager.start
-        end
-
-        config.on(:shutdown) do
-          SidekiqUniqueJobs::Orphans::Manager.stop
-        end
-      end
-    end
-
-    #
-    # Configures the Sidekiq client
-    #
-    def self.configure_client
-      Sidekiq.configure_client do |config|
-        config.client_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Client
-          end
-        end
-      end
-    end
-
     # The sidekiq job hash
     # @return [Hash] the Sidekiq job hash
     attr_reader :item

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -11,9 +11,9 @@ module SidekiqUniqueJobs
       #   @return [String] rthe sidekiq queue this job belongs to
       attr_reader :queue
       #
-      # @!attribute [r] unique_digest
+      # @!attribute [r] lock_digest
       #   @return [String] the unique digest to use for locking
-      attr_reader :unique_digest
+      attr_reader :lock_digest
 
       #
       # Initialize a new Replace strategy
@@ -22,8 +22,8 @@ module SidekiqUniqueJobs
       #
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @queue         = item[QUEUE]
-        @unique_digest = item[LOCK_DIGEST]
+        @queue       = item[QUEUE]
+        @lock_digest = item[LOCK_DIGEST]
       end
 
       #
@@ -37,10 +37,11 @@ module SidekiqUniqueJobs
       def call(&block)
         return unless (deleted_job = delete_job_by_digest)
 
-        log_info("Deleting job: #{deleted_job}")
+        log_info("Deleted job: #{deleted_job}")
         if (del_count = delete_lock)
-          log_info("Deleted `#{del_count}` keys for #{unique_digest}")
+          log_info("Deleted `#{del_count}` keys for #{lock_digest}")
         end
+
         block&.call
       end
 
@@ -54,7 +55,7 @@ module SidekiqUniqueJobs
       def delete_job_by_digest
         call_script(:delete_job_by_digest,
                     keys: ["#{QUEUE}:#{queue}", SCHEDULE, RETRY],
-                    argv: [unique_digest])
+                    argv: [lock_digest])
       end
 
       #
@@ -64,7 +65,7 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of keys deleted
       #
       def delete_lock
-        digests.delete_by_digest(unique_digest)
+        digests.delete_by_digest(lock_digest)
       end
 
       #

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
       def call
         if sidekiq_worker_class?
           log_info("Rescheduling #{item[LOCK_DIGEST]}")
-          worker_class&.perform_in(5, *item[ARGS])
+          worker_class.perform_in(5, *item[ARGS])
         else
           log_warn("Skip rescheduling of #{item[LOCK_DIGEST]} because #{worker_class} is not a Sidekiq::Worker")
         end

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
           :reap_orphans,
           conn,
           keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
-          argv: [reaper_count],
+          argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end
     end

data/lib/sidekiq_unique_jobs/orphans/reaper.rb

@@ -76,6 +76,16 @@ module SidekiqUniqueJobs
         config.reaper
       end
 
+      #
+      # The configured timeout for the reaper
+      #
+      #
+      # @return [Integer] timeout in seconds
+      #
+      def reaper_timeout
+        config.reaper_timeout
+      end
+
       #
       # The number of locks to reap at a time
       #

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -117,7 +117,7 @@ module SidekiqUniqueJobs
         end
       end
 
-      def active?(digest) # rubocop:disable Metrics/MethodLength
+      def active?(digest) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
         Sidekiq.redis do |conn|
           procs = conn.sscan_each("processes").to_a
           return false if procs.empty?
@@ -132,7 +132,10 @@ module SidekiqUniqueJobs
             next unless workers.any?
 
             workers.each_pair do |_tid, job|
-              return true if load_json(job)[LOCK_DIGEST] == digest
+              item = load_json(job)
+
+              return true if item.dig(PAYLOAD, LOCK_DIGEST) == digest
+              return true if considered_active?(item[CREATED_AT])
             end
           end
 
@@ -140,6 +143,10 @@ module SidekiqUniqueJobs
         end
       end
 
+      def considered_active?(time_f)
+        (Time.now - reaper_timeout).to_f < time_f
+      end
+
       #
       # Loops through all the redis queues and yields them one by one
       #