sidekiq-unique-jobs 7.0.0.beta28 → 7.0.3

data/lib/sidekiq-unique-jobs.rb

@@ -1,5 +1,3 @@
 # frozen_string_literal: true
 
 require "sidekiq_unique_jobs"
-
-SidekiqUniqueJobs::Middleware.configure

data/lib/sidekiq_unique_jobs.rb

@@ -3,9 +3,10 @@
 require "brpoplpush/redis_script"
 require "concurrent/future"
 require "concurrent/promises"
-require "concurrent/timer_task"
 require "concurrent/map"
 require "concurrent/mutable_struct"
+require "concurrent/timer_task"
+require "concurrent/executor/ruby_single_thread_executor"
 require "digest"
 require "digest/sha1"
 require "erb"
@@ -14,6 +15,7 @@ require "json"
 require "pathname"
 require "sidekiq"
 
+require "sidekiq_unique_jobs/timer_task"
 require "sidekiq_unique_jobs/version"
 require "sidekiq_unique_jobs/version_check"
 require "sidekiq_unique_jobs/constants"
@@ -71,3 +73,4 @@ require "sidekiq_unique_jobs/config"
 require "sidekiq_unique_jobs/sidekiq_unique_jobs"
 require "sidekiq_unique_jobs/update_version"
 require "sidekiq_unique_jobs/upgrade_locks"
+require "sidekiq_unique_jobs/server"

data/lib/sidekiq_unique_jobs/batch_delete.rb

@@ -44,7 +44,7 @@ module SidekiqUniqueJobs
     #
     # @return [void]
     #
-    def self.call(digests, conn)
+    def self.call(digests, conn = nil)
       new(digests, conn).call
     end
 

data/lib/sidekiq_unique_jobs/changelog.rb

@@ -7,6 +7,13 @@ module SidekiqUniqueJobs
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   class Changelog < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
+    DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
+    SCAN_PATTERN  = "*"
+
     def initialize
       super(CHANGELOGS)
     end
@@ -34,10 +41,10 @@ module SidekiqUniqueJobs
     #
     # @return [Array<Hash>] an array of entries
     #
-    def entries(pattern: "*", count: nil)
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
       options = {}
-      options[:match] = pattern if pattern
-      options[:count] = count if count
+      options[:match] = pattern
+      options[:count] = count
 
       redis do |conn|
         conn.zscan_each(key, **options).to_a.map { |entry| load_json(entry[0]) }
@@ -53,7 +60,7 @@ module SidekiqUniqueJobs
     #
     # @return [Array<Integer, Integer, Array<Hash>] the total size, next cursor and changelog entries
     #
-    def page(cursor, pattern: "*", page_size: 100)
+    def page(cursor: 0, pattern: "*", page_size: 100)
       redis do |conn|
         total_size, result = conn.multi do
           conn.zcard(key)

data/lib/sidekiq_unique_jobs/constants.rb

@@ -11,6 +11,7 @@ module SidekiqUniqueJobs
   AT                    ||= "at"
   CHANGELOGS            ||= "uniquejobs:changelog"
   CLASS                 ||= "class"
+  CREATED_AT            ||= "created_at"
   DEAD_VERSION          ||= "uniquejobs:dead"
   DIGESTS               ||= "uniquejobs:digests"
   ERRORS                ||= "errors"

data/lib/sidekiq_unique_jobs/digests.rb

@@ -78,7 +78,7 @@ module SidekiqUniqueJobs
     def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
       options = {}
       options[:match] = pattern
-      options[:count] = count if count
+      options[:count] = count
 
       result = redis { |conn| conn.zscan_each(key, **options).to_a }
 

data/lib/sidekiq_unique_jobs/json.rb

@@ -15,11 +15,17 @@ module SidekiqUniqueJobs
     # @return [Object]
     #
     def load_json(string)
-      return unless string && !string.empty?
+      return if string.nil? || string.empty?
 
       ::JSON.parse(string)
     end
 
+    def safe_load_json(string)
+      return string if string.is_a?(Hash)
+
+      load_json(string)
+    end
+
     #
     # Dumps an object into a JSON string
     #

data/lib/sidekiq_unique_jobs/lock.rb

@@ -53,7 +53,7 @@ module SidekiqUniqueJobs
     #
     # Locks a job_id
     #
-    # @note intended only for testing purposez
+    # @note intended only for testing purposes
     #
     # @param [String] job_id a sidekiq JID
     # @param [Hash] lock_info information about the lock
@@ -73,6 +73,36 @@ module SidekiqUniqueJobs
       end
     end
 
+    #
+    # Create the :QUEUED key
+    #
+    # @note intended only for testing purposes
+    #
+    # @param [String] job_id a sidekiq JID
+    #
+    # @return [void]
+    #
+    def queue(job_id)
+      redis do |conn|
+        conn.lpush(key.queued, job_id)
+      end
+    end
+
+    #
+    # Create the :PRIMED key
+    #
+    # @note intended only for testing purposes
+    #
+    # @param [String] job_id a sidekiq JID
+    #
+    # @return [void]
+    #
+    def prime(job_id)
+      redis do |conn|
+        conn.lpush(key.primed, job_id)
+      end
+    end
+
     #
     # Unlock a specific job_id
     #

data/lib/sidekiq_unique_jobs/lock_config.rb

@@ -98,6 +98,8 @@ module SidekiqUniqueJobs
     # @return [String]
     #
     def errors_as_string
+      return if valid?
+
       @errors_as_string ||= begin
         error_msg = +"\t"
         error_msg << errors.map { |key, val| "#{key}: :#{val}" }.join("\n\t")

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -69,25 +69,24 @@ redis.call("LREM", queued, -1, job_id)
 log_debug("LREM", primed, 1, job_id)
 redis.call("LREM", primed, 1, job_id)
 
--- The Sidekiq client should only set pttl for until_expired
--- The Sidekiq server should set pttl for all other jobs
+-- The Sidekiq client sets pttl
 if pttl and pttl > 0 then
   log_debug("PEXPIRE", digest, pttl)
   redis.call("PEXPIRE", digest, pttl)
 
-  log_debug("PEXPIRE", queued, pttl)
-  redis.call("PEXPIRE", queued, pttl)
-
-  log_debug("PEXPIRE", primed, pttl)
-  redis.call("PEXPIRE", primed, pttl)
-
   log_debug("PEXPIRE", locked, pttl)
   redis.call("PEXPIRE", locked, pttl)
-
-  log_debug("PEXPIRE", info, pttl)
-  redis.call("PEXPIRE", info, pttl)
 end
 
+log_debug("PEXPIRE", queued, 1000)
+redis.call("PEXPIRE", queued, 1000)
+
+log_debug("PEXPIRE", primed, 1000)
+redis.call("PEXPIRE", primed, 1000)
+
+log_debug("PEXPIRE", info, 1000)
+redis.call("PEXPIRE", info, 1000)
+
 log("Locked")
 log_debug("END lock digest:", digest, "job_id:", job_id)
 return job_id

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -7,15 +7,16 @@ local retry_set    = KEYS[3]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
-local reaper_count  = tonumber(ARGV[1])
+local reaper_count = tonumber(ARGV[1])
+local threshold    = tonumber(ARGV[2])
 --------  END argv  ---------
 
 --------  BEGIN injected arguments --------
-local current_time = tonumber(ARGV[2])
-local debug_lua    = ARGV[3] == "true"
-local max_history  = tonumber(ARGV[4])
-local script_name  = ARGV[5] .. ".lua"
-local redisversion = ARGV[6]
+local current_time = tonumber(ARGV[3])
+local debug_lua    = ARGV[4] == "true"
+local max_history  = tonumber(ARGV[5])
+local script_name  = ARGV[6] .. ".lua"
+local redisversion = ARGV[7]
 ---------  END injected arguments ---------
 
 
@@ -65,7 +66,7 @@ repeat
     -- TODO: Add check for jobs checked out by process
     if found ~= true then
       log_debug("Searching for digest:", digest, "in process sets")
-      found = find_digest_in_process_set(digest)
+      found = find_digest_in_process_set(digest, threshold)
     end
 
     if found ~= true then

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua

@@ -1,4 +1,4 @@
-local function find_digest_in_process_set(digest)
+local function find_digest_in_process_set(digest, threshold)
   local process_cursor = 0
   local job_cursor     = 0
   local pattern        = "*" .. digest .. "*"
@@ -26,11 +26,18 @@ local function find_digest_in_process_set(digest)
         log_debug("No entries in:", workers_key)
       else
         for i = 1, #jobs, 2 do
-          if string.find(jobs[i +1], digest) then
+          local jobstr = jobs[i +1]
+          if string.find(jobstr, digest) then
             log_debug("Found digest", digest, "in:", workers_key)
             found = true
             break
           end
+
+          local job = cjson.decode(jobstr)
+          if job.payload.created_at > threshold then
+            found = true
+            break
+          end
         end
       end
 

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -11,63 +11,6 @@ module SidekiqUniqueJobs
     include SidekiqUniqueJobs::OptionsWithFallback
     include SidekiqUniqueJobs::JSON
 
-    #
-    # Configure both server and client
-    #
-    def self.configure
-      configure_server
-      configure_client
-    end
-
-    #
-    # Configures the Sidekiq server
-    #
-    def self.configure_server # rubocop:disable Metrics/MethodLength
-      Sidekiq.configure_server do |config|
-        config.client_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Client
-          end
-        end
-
-        config.server_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Server)
-            chain.insert_after Apartment::Sidekiq::Middleware::Server, SidekiqUniqueJobs::Middleware::Server
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Server
-          end
-        end
-
-        config.on(:startup) do
-          SidekiqUniqueJobs::UpdateVersion.call
-          SidekiqUniqueJobs::UpgradeLocks.call
-
-          SidekiqUniqueJobs::Orphans::Manager.start
-        end
-
-        config.on(:shutdown) do
-          SidekiqUniqueJobs::Orphans::Manager.stop
-        end
-      end
-    end
-
-    #
-    # Configures the Sidekiq client
-    #
-    def self.configure_client
-      Sidekiq.configure_client do |config|
-        config.client_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Client
-          end
-        end
-      end
-    end
-
     # The sidekiq job hash
     # @return [Hash] the Sidekiq job hash
     attr_reader :item

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -11,9 +11,9 @@ module SidekiqUniqueJobs
       #   @return [String] rthe sidekiq queue this job belongs to
       attr_reader :queue
       #
-      # @!attribute [r] unique_digest
+      # @!attribute [r] lock_digest
       #   @return [String] the unique digest to use for locking
-      attr_reader :unique_digest
+      attr_reader :lock_digest
 
       #
       # Initialize a new Replace strategy
@@ -22,8 +22,8 @@ module SidekiqUniqueJobs
       #
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @queue         = item[QUEUE]
-        @unique_digest = item[LOCK_DIGEST]
+        @queue       = item[QUEUE]
+        @lock_digest = item[LOCK_DIGEST]
       end
 
       #
@@ -37,10 +37,11 @@ module SidekiqUniqueJobs
       def call(&block)
         return unless (deleted_job = delete_job_by_digest)
 
-        log_info("Deleting job: #{deleted_job}")
+        log_info("Deleted job: #{deleted_job}")
         if (del_count = delete_lock)
-          log_info("Deleted `#{del_count}` keys for #{unique_digest}")
+          log_info("Deleted `#{del_count}` keys for #{lock_digest}")
         end
+
         block&.call
       end
 
@@ -54,7 +55,7 @@ module SidekiqUniqueJobs
       def delete_job_by_digest
         call_script(:delete_job_by_digest,
                     keys: ["#{QUEUE}:#{queue}", SCHEDULE, RETRY],
-                    argv: [unique_digest])
+                    argv: [lock_digest])
       end
 
       #
@@ -64,7 +65,7 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of keys deleted
       #
       def delete_lock
-        digests.delete_by_digest(unique_digest)
+        digests.delete_by_digest(lock_digest)
       end
 
       #

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
       def call
         if sidekiq_worker_class?
           log_info("Rescheduling #{item[LOCK_DIGEST]}")
-          worker_class&.perform_in(5, *item[ARGS])
+          worker_class.perform_in(5, *item[ARGS])
         else
           log_warn("Skip rescheduling of #{item[LOCK_DIGEST]} because #{worker_class} is not a Sidekiq::Worker")
         end

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
           :reap_orphans,
           conn,
           keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
-          argv: [reaper_count],
+          argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end
     end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -20,12 +20,14 @@ module SidekiqUniqueJobs
       # Starts a separate thread that periodically reaps orphans
       #
       #
-      # @return [Concurrent::TimerTask] the task that was started
+      # @return [SidekiqUniqueJobs::TimerTask] the task that was started
       #
-      def start # rubocop:disable
+      def start(test_task = nil) # rubocop:disable
         return if disabled?
         return if registered?
 
+        self.task = test_task || default_task
+
         with_logging_context do
           register_reaper_process
           log_info("Starting Reaper")
@@ -59,7 +61,11 @@ module SidekiqUniqueJobs
       # @return [<type>] <description>
       #
       def task
-        @task ||= Concurrent::TimerTask.new(timer_task_options) do
+        @task ||= default_task
+      end
+
+      def default_task
+        SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
           with_logging_context do
             redis do |conn|
               refresh_reaper_mutex
@@ -69,6 +75,10 @@ module SidekiqUniqueJobs
         end
       end
 
+      def task=(task)
+        @task = task
+      end
+
       #
       # Arguments passed on to the timer task
       #