sidekiq-unique-jobs 7.0.0.beta27 → 7.0.2

data/lib/sidekiq-unique-jobs.rb

@@ -1,5 +1,3 @@
 # frozen_string_literal: true
 
 require "sidekiq_unique_jobs"
-
-SidekiqUniqueJobs::Middleware.configure

data/lib/sidekiq_unique_jobs.rb

@@ -71,3 +71,4 @@ require "sidekiq_unique_jobs/config"
 require "sidekiq_unique_jobs/sidekiq_unique_jobs"
 require "sidekiq_unique_jobs/update_version"
 require "sidekiq_unique_jobs/upgrade_locks"
+require "sidekiq_unique_jobs/server"

data/lib/sidekiq_unique_jobs/batch_delete.rb

@@ -44,7 +44,7 @@ module SidekiqUniqueJobs
     #
     # @return [void]
     #
-    def self.call(digests, conn)
+    def self.call(digests, conn = nil)
       new(digests, conn).call
     end
 

data/lib/sidekiq_unique_jobs/changelog.rb

@@ -7,6 +7,13 @@ module SidekiqUniqueJobs
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   class Changelog < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
+    DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
+    SCAN_PATTERN  = "*"
+
     def initialize
       super(CHANGELOGS)
     end
@@ -34,10 +41,10 @@ module SidekiqUniqueJobs
     #
     # @return [Array<Hash>] an array of entries
     #
-    def entries(pattern: "*", count: nil)
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
       options = {}
-      options[:match] = pattern if pattern
-      options[:count] = count if count
+      options[:match] = pattern
+      options[:count] = count
 
       redis do |conn|
         conn.zscan_each(key, **options).to_a.map { |entry| load_json(entry[0]) }
@@ -53,7 +60,7 @@ module SidekiqUniqueJobs
     #
     # @return [Array<Integer, Integer, Array<Hash>] the total size, next cursor and changelog entries
     #
-    def page(cursor, pattern: "*", page_size: 100)
+    def page(cursor: 0, pattern: "*", page_size: 100)
       redis do |conn|
         total_size, result = conn.multi do
           conn.zcard(key)

data/lib/sidekiq_unique_jobs/config.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
                                                    :lock_timeout,
                                                    :lock_ttl,
                                                    :enabled,
-                                                   :unique_prefix,
+                                                   :lock_prefix,
                                                    :logger,
                                                    :locks,
                                                    :strategies,
@@ -127,7 +127,7 @@ module SidekiqUniqueJobs
     #   default_lock_timeout: 0,
     #   default_lock_ttl: nil,
     #   enabled: true,
-    #   unique_prefix: "uniquejobs",
+    #   lock_prefix: "uniquejobs",
     #   logger: #<Sidekiq::Logger:0x00007f81e096b0e0 @level=1 ...>,
     #   locks: {
     #     around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,

data/lib/sidekiq_unique_jobs/constants.rb

@@ -11,6 +11,7 @@ module SidekiqUniqueJobs
   AT                    ||= "at"
   CHANGELOGS            ||= "uniquejobs:changelog"
   CLASS                 ||= "class"
+  CREATED_AT            ||= "created_at"
   DEAD_VERSION          ||= "uniquejobs:dead"
   DIGESTS               ||= "uniquejobs:digests"
   ERRORS                ||= "errors"
@@ -32,6 +33,7 @@ module SidekiqUniqueJobs
   ON_CLIENT_CONFLICT    ||= "on_client_conflict"
   ON_CONFLICT           ||= "on_conflict"
   ON_SERVER_CONFLICT    ||= "on_server_conflict"
+  PAYLOAD               ||= "payload"
   PROCESSES             ||= "processes"
   QUEUE                 ||= "queue"
   RETRY                 ||= "retry"

data/lib/sidekiq_unique_jobs/digests.rb

@@ -78,7 +78,7 @@ module SidekiqUniqueJobs
     def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
       options = {}
       options[:match] = pattern
-      options[:count] = count if count
+      options[:count] = count
 
       result = redis { |conn| conn.zscan_each(key, **options).to_a }
 

data/lib/sidekiq_unique_jobs/job.rb

@@ -52,7 +52,7 @@ module SidekiqUniqueJobs
     end
 
     def add_lock_prefix(item)
-      item[LOCK_PREFIX] ||= SidekiqUniqueJobs.config.unique_prefix
+      item[LOCK_PREFIX] ||= SidekiqUniqueJobs.config.lock_prefix
     end
   end
 end

data/lib/sidekiq_unique_jobs/json.rb

@@ -15,11 +15,17 @@ module SidekiqUniqueJobs
     # @return [Object]
     #
     def load_json(string)
-      return unless string && !string.empty?
+      return if string.nil? || string.empty?
 
       ::JSON.parse(string)
     end
 
+    def safe_load_json(string)
+      return string if string.is_a?(Hash)
+
+      load_json(string)
+    end
+
     #
     # Dumps an object into a JSON string
     #

data/lib/sidekiq_unique_jobs/lock.rb

@@ -53,7 +53,7 @@ module SidekiqUniqueJobs
     #
     # Locks a job_id
     #
-    # @note intended only for testing purposez
+    # @note intended only for testing purposes
     #
     # @param [String] job_id a sidekiq JID
     # @param [Hash] lock_info information about the lock
@@ -73,6 +73,36 @@ module SidekiqUniqueJobs
       end
     end
 
+    #
+    # Create the :QUEUED key
+    #
+    # @note intended only for testing purposes
+    #
+    # @param [String] job_id a sidekiq JID
+    #
+    # @return [void]
+    #
+    def queue(job_id)
+      redis do |conn|
+        conn.lpush(key.queued, job_id)
+      end
+    end
+
+    #
+    # Create the :PRIMED key
+    #
+    # @note intended only for testing purposes
+    #
+    # @param [String] job_id a sidekiq JID
+    #
+    # @return [void]
+    #
+    def prime(job_id)
+      redis do |conn|
+        conn.lpush(key.primed, job_id)
+      end
+    end
+
     #
     # Unlock a specific job_id
     #

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -43,7 +43,7 @@ module SidekiqUniqueJobs
           end
         end
       ensure
-        locksmith.unlock
+        locksmith.unlock!
       end
 
       private

data/lib/sidekiq_unique_jobs/lock_config.rb

@@ -98,6 +98,8 @@ module SidekiqUniqueJobs
     # @return [String]
     #
     def errors_as_string
+      return if valid?
+
       @errors_as_string ||= begin
         error_msg = +"\t"
         error_msg << errors.map { |key, val| "#{key}: :#{val}" }.join("\n\t")

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -87,7 +87,7 @@ module SidekiqUniqueJobs
     #
     def lock(&block)
       redis(redis_pool) do |conn|
-        return lock_async(conn, &block) if block_given?
+        return lock_async(conn, &block) if block
 
         lock_sync(conn) do
           return job_id

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -69,25 +69,24 @@ redis.call("LREM", queued, -1, job_id)
 log_debug("LREM", primed, 1, job_id)
 redis.call("LREM", primed, 1, job_id)
 
--- The Sidekiq client should only set pttl for until_expired
--- The Sidekiq server should set pttl for all other jobs
+-- The Sidekiq client sets pttl
 if pttl and pttl > 0 then
   log_debug("PEXPIRE", digest, pttl)
   redis.call("PEXPIRE", digest, pttl)
 
-  log_debug("PEXPIRE", queued, pttl)
-  redis.call("PEXPIRE", queued, pttl)
-
-  log_debug("PEXPIRE", primed, pttl)
-  redis.call("PEXPIRE", primed, pttl)
-
   log_debug("PEXPIRE", locked, pttl)
   redis.call("PEXPIRE", locked, pttl)
-
-  log_debug("PEXPIRE", info, pttl)
-  redis.call("PEXPIRE", info, pttl)
 end
 
+log_debug("PEXPIRE", queued, 1000)
+redis.call("PEXPIRE", queued, 1000)
+
+log_debug("PEXPIRE", primed, 1000)
+redis.call("PEXPIRE", primed, 1000)
+
+log_debug("PEXPIRE", info, 1000)
+redis.call("PEXPIRE", info, 1000)
+
 log("Locked")
 log_debug("END lock digest:", digest, "job_id:", job_id)
 return job_id

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -7,15 +7,16 @@ local retry_set    = KEYS[3]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
-local reaper_count  = tonumber(ARGV[1])
+local reaper_count = tonumber(ARGV[1])
+local threshold    = tonumber(ARGV[2])
 --------  END argv  ---------
 
 --------  BEGIN injected arguments --------
-local current_time = tonumber(ARGV[2])
-local debug_lua    = ARGV[3] == "true"
-local max_history  = tonumber(ARGV[4])
-local script_name  = ARGV[5] .. ".lua"
-local redisversion = ARGV[6]
+local current_time = tonumber(ARGV[3])
+local debug_lua    = ARGV[4] == "true"
+local max_history  = tonumber(ARGV[5])
+local script_name  = ARGV[6] .. ".lua"
+local redisversion = ARGV[7]
 ---------  END injected arguments ---------
 
 
@@ -65,7 +66,7 @@ repeat
     -- TODO: Add check for jobs checked out by process
     if found ~= true then
       log_debug("Searching for digest:", digest, "in process sets")
-      found = find_digest_in_process_set(digest)
+      found = find_digest_in_process_set(digest, threshold)
     end
 
     if found ~= true then

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua

@@ -1,4 +1,4 @@
-local function find_digest_in_process_set(digest)
+local function find_digest_in_process_set(digest, threshold)
   local process_cursor = 0
   local job_cursor     = 0
   local pattern        = "*" .. digest .. "*"
@@ -26,11 +26,18 @@ local function find_digest_in_process_set(digest)
         log_debug("No entries in:", workers_key)
       else
         for i = 1, #jobs, 2 do
-          if string.find(jobs[i +1], digest) then
+          local jobstr = jobs[i +1]
+          if string.find(jobstr, digest) then
             log_debug("Found digest", digest, "in:", workers_key)
             found = true
             break
           end
+
+          local job = cjson.decode(jobstr)
+          if job.payload.created_at > threshold then
+            found = true
+            break
+          end
         end
       end
 

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -11,63 +11,6 @@ module SidekiqUniqueJobs
     include SidekiqUniqueJobs::OptionsWithFallback
     include SidekiqUniqueJobs::JSON
 
-    #
-    # Configure both server and client
-    #
-    def self.configure
-      configure_server
-      configure_client
-    end
-
-    #
-    # Configures the Sidekiq server
-    #
-    def self.configure_server # rubocop:disable Metrics/MethodLength
-      Sidekiq.configure_server do |config|
-        config.client_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Client
-          end
-        end
-
-        config.server_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Server)
-            chain.insert_after Apartment::Sidekiq::Middleware::Server, SidekiqUniqueJobs::Middleware::Server
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Server
-          end
-        end
-
-        config.on(:startup) do
-          SidekiqUniqueJobs::UpdateVersion.call
-          SidekiqUniqueJobs::UpgradeLocks.call
-
-          SidekiqUniqueJobs::Orphans::Manager.start
-        end
-
-        config.on(:shutdown) do
-          SidekiqUniqueJobs::Orphans::Manager.stop
-        end
-      end
-    end
-
-    #
-    # Configures the Sidekiq client
-    #
-    def self.configure_client
-      Sidekiq.configure_client do |config|
-        config.client_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Client
-          end
-        end
-      end
-    end
-
     # The sidekiq job hash
     # @return [Hash] the Sidekiq job hash
     attr_reader :item

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -11,9 +11,9 @@ module SidekiqUniqueJobs
       #   @return [String] rthe sidekiq queue this job belongs to
       attr_reader :queue
       #
-      # @!attribute [r] unique_digest
+      # @!attribute [r] lock_digest
       #   @return [String] the unique digest to use for locking
-      attr_reader :unique_digest
+      attr_reader :lock_digest
 
       #
       # Initialize a new Replace strategy
@@ -22,8 +22,8 @@ module SidekiqUniqueJobs
       #
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @queue         = item[QUEUE]
-        @unique_digest = item[LOCK_DIGEST]
+        @queue       = item[QUEUE]
+        @lock_digest = item[LOCK_DIGEST]
       end
 
       #
@@ -37,10 +37,11 @@ module SidekiqUniqueJobs
       def call(&block)
         return unless (deleted_job = delete_job_by_digest)
 
-        log_info("Deleting job: #{deleted_job}")
+        log_info("Deleted job: #{deleted_job}")
         if (del_count = delete_lock)
-          log_info("Deleted `#{del_count}` keys for #{unique_digest}")
+          log_info("Deleted `#{del_count}` keys for #{lock_digest}")
         end
+
         block&.call
       end
 
@@ -54,7 +55,7 @@ module SidekiqUniqueJobs
       def delete_job_by_digest
         call_script(:delete_job_by_digest,
                     keys: ["#{QUEUE}:#{queue}", SCHEDULE, RETRY],
-                    argv: [unique_digest])
+                    argv: [lock_digest])
       end
 
       #
@@ -64,7 +65,7 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of keys deleted
       #
       def delete_lock
-        digests.delete_by_digest(unique_digest)
+        digests.delete_by_digest(lock_digest)
       end
 
       #

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
       def call
         if sidekiq_worker_class?
           log_info("Rescheduling #{item[LOCK_DIGEST]}")
-          worker_class&.perform_in(5, *item[ARGS])
+          worker_class.perform_in(5, *item[ARGS])
         else
           log_warn("Skip rescheduling of #{item[LOCK_DIGEST]} because #{worker_class} is not a Sidekiq::Worker")
         end

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
           :reap_orphans,
           conn,
           keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
-          argv: [reaper_count],
+          argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end
     end