sidekiq-unique-jobs 7.0.0.beta29 → 7.0.4

data/lib/sidekiq_unique_jobs/lock_config.rb

@@ -98,6 +98,8 @@ module SidekiqUniqueJobs
     # @return [String]
     #
     def errors_as_string
+      return if valid?
+
       @errors_as_string ||= begin
         error_msg = +"\t"
         error_msg << errors.map { |key, val| "#{key}: :#{val}" }.join("\n\t")

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -69,25 +69,24 @@ redis.call("LREM", queued, -1, job_id)
 log_debug("LREM", primed, 1, job_id)
 redis.call("LREM", primed, 1, job_id)
 
--- The Sidekiq client should only set pttl for until_expired
--- The Sidekiq server should set pttl for all other jobs
+-- The Sidekiq client sets pttl
 if pttl and pttl > 0 then
   log_debug("PEXPIRE", digest, pttl)
   redis.call("PEXPIRE", digest, pttl)
 
-  log_debug("PEXPIRE", queued, pttl)
-  redis.call("PEXPIRE", queued, pttl)
-
-  log_debug("PEXPIRE", primed, pttl)
-  redis.call("PEXPIRE", primed, pttl)
-
   log_debug("PEXPIRE", locked, pttl)
   redis.call("PEXPIRE", locked, pttl)
-
-  log_debug("PEXPIRE", info, pttl)
-  redis.call("PEXPIRE", info, pttl)
 end
 
+log_debug("PEXPIRE", queued, 1000)
+redis.call("PEXPIRE", queued, 1000)
+
+log_debug("PEXPIRE", primed, 1000)
+redis.call("PEXPIRE", primed, 1000)
+
+log_debug("PEXPIRE", info, 1000)
+redis.call("PEXPIRE", info, 1000)
+
 log("Locked")
 log_debug("END lock digest:", digest, "job_id:", job_id)
 return job_id

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -7,15 +7,16 @@ local retry_set    = KEYS[3]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
-local reaper_count  = tonumber(ARGV[1])
+local reaper_count = tonumber(ARGV[1])
+local threshold    = tonumber(ARGV[2])
 --------  END argv  ---------
 
 --------  BEGIN injected arguments --------
-local current_time = tonumber(ARGV[2])
-local debug_lua    = ARGV[3] == "true"
-local max_history  = tonumber(ARGV[4])
-local script_name  = ARGV[5] .. ".lua"
-local redisversion = ARGV[6]
+local current_time = tonumber(ARGV[3])
+local debug_lua    = ARGV[4] == "true"
+local max_history  = tonumber(ARGV[5])
+local script_name  = ARGV[6] .. ".lua"
+local redisversion = ARGV[7]
 ---------  END injected arguments ---------
 
 
@@ -65,7 +66,7 @@ repeat
     -- TODO: Add check for jobs checked out by process
     if found ~= true then
       log_debug("Searching for digest:", digest, "in process sets")
-      found = find_digest_in_process_set(digest)
+      found = find_digest_in_process_set(digest, threshold)
     end
 
     if found ~= true then

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua

@@ -1,4 +1,4 @@
-local function find_digest_in_process_set(digest)
+local function find_digest_in_process_set(digest, threshold)
   local process_cursor = 0
   local job_cursor     = 0
   local pattern        = "*" .. digest .. "*"
@@ -26,11 +26,18 @@ local function find_digest_in_process_set(digest)
         log_debug("No entries in:", workers_key)
       else
         for i = 1, #jobs, 2 do
-          if string.find(jobs[i +1], digest) then
+          local jobstr = jobs[i +1]
+          if string.find(jobstr, digest) then
             log_debug("Found digest", digest, "in:", workers_key)
             found = true
             break
           end
+
+          local job = cjson.decode(jobstr)
+          if job.payload.created_at > threshold then
+            found = true
+            break
+          end
         end
       end
 

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -11,63 +11,6 @@ module SidekiqUniqueJobs
     include SidekiqUniqueJobs::OptionsWithFallback
     include SidekiqUniqueJobs::JSON
 
-    #
-    # Configure both server and client
-    #
-    def self.configure
-      configure_server
-      configure_client
-    end
-
-    #
-    # Configures the Sidekiq server
-    #
-    def self.configure_server # rubocop:disable Metrics/MethodLength
-      Sidekiq.configure_server do |config|
-        config.client_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Client
-          end
-        end
-
-        config.server_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Server)
-            chain.insert_after Apartment::Sidekiq::Middleware::Server, SidekiqUniqueJobs::Middleware::Server
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Server
-          end
-        end
-
-        config.on(:startup) do
-          SidekiqUniqueJobs::UpdateVersion.call
-          SidekiqUniqueJobs::UpgradeLocks.call
-
-          SidekiqUniqueJobs::Orphans::Manager.start
-        end
-
-        config.on(:shutdown) do
-          SidekiqUniqueJobs::Orphans::Manager.stop
-        end
-      end
-    end
-
-    #
-    # Configures the Sidekiq client
-    #
-    def self.configure_client
-      Sidekiq.configure_client do |config|
-        config.client_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Client
-          end
-        end
-      end
-    end
-
     # The sidekiq job hash
     # @return [Hash] the Sidekiq job hash
     attr_reader :item

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -11,9 +11,9 @@ module SidekiqUniqueJobs
       #   @return [String] rthe sidekiq queue this job belongs to
       attr_reader :queue
       #
-      # @!attribute [r] unique_digest
+      # @!attribute [r] lock_digest
       #   @return [String] the unique digest to use for locking
-      attr_reader :unique_digest
+      attr_reader :lock_digest
 
       #
       # Initialize a new Replace strategy
@@ -22,8 +22,8 @@ module SidekiqUniqueJobs
       #
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @queue         = item[QUEUE]
-        @unique_digest = item[LOCK_DIGEST]
+        @queue       = item[QUEUE]
+        @lock_digest = item[LOCK_DIGEST]
       end
 
       #
@@ -37,10 +37,11 @@ module SidekiqUniqueJobs
       def call(&block)
         return unless (deleted_job = delete_job_by_digest)
 
-        log_info("Deleting job: #{deleted_job}")
+        log_info("Deleted job: #{deleted_job}")
         if (del_count = delete_lock)
-          log_info("Deleted `#{del_count}` keys for #{unique_digest}")
+          log_info("Deleted `#{del_count}` keys for #{lock_digest}")
         end
+
         block&.call
       end
 
@@ -54,7 +55,7 @@ module SidekiqUniqueJobs
       def delete_job_by_digest
         call_script(:delete_job_by_digest,
                     keys: ["#{QUEUE}:#{queue}", SCHEDULE, RETRY],
-                    argv: [unique_digest])
+                    argv: [lock_digest])
       end
 
       #
@@ -64,7 +65,7 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of keys deleted
       #
       def delete_lock
-        digests.delete_by_digest(unique_digest)
+        digests.delete_by_digest(lock_digest)
       end
 
       #

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
       def call
         if sidekiq_worker_class?
           log_info("Rescheduling #{item[LOCK_DIGEST]}")
-          worker_class&.perform_in(5, *item[ARGS])
+          worker_class.perform_in(5, *item[ARGS])
         else
           log_warn("Skip rescheduling of #{item[LOCK_DIGEST]} because #{worker_class} is not a Sidekiq::Worker")
         end

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
           :reap_orphans,
           conn,
           keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
-          argv: [reaper_count],
+          argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end
     end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -20,12 +20,14 @@ module SidekiqUniqueJobs
       # Starts a separate thread that periodically reaps orphans
       #
       #
-      # @return [Concurrent::TimerTask] the task that was started
+      # @return [SidekiqUniqueJobs::TimerTask] the task that was started
       #
-      def start # rubocop:disable
+      def start(test_task = nil) # rubocop:disable
         return if disabled?
         return if registered?
 
+        self.task = test_task || default_task
+
         with_logging_context do
           register_reaper_process
           log_info("Starting Reaper")
@@ -59,7 +61,11 @@ module SidekiqUniqueJobs
       # @return [<type>] <description>
       #
       def task
-        @task ||= Concurrent::TimerTask.new(timer_task_options) do
+        @task ||= default_task
+      end
+
+      def default_task
+        SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
           with_logging_context do
             redis do |conn|
               refresh_reaper_mutex
@@ -69,6 +75,10 @@ module SidekiqUniqueJobs
         end
       end
 
+      def task=(task)
+        @task = task
+      end
+
       #
       # Arguments passed on to the timer task
       #

data/lib/sidekiq_unique_jobs/orphans/reaper.rb

@@ -76,6 +76,16 @@ module SidekiqUniqueJobs
         config.reaper
       end
 
+      #
+      # The configured timeout for the reaper
+      #
+      #
+      # @return [Integer] timeout in seconds
+      #
+      def reaper_timeout
+        config.reaper_timeout
+      end
+
       #
       # The number of locks to reap at a time
       #

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -52,11 +52,11 @@ module SidekiqUniqueJobs
       # @return [Array<String>] an array of orphaned digests
       #
       def orphans
-        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, result|
+        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, memo|
           next if belongs_to_job?(digest)
 
-          result << digest
-          break if result.size >= reaper_count
+          memo << digest
+          break if memo.size >= reaper_count
         end
       end
 
@@ -117,7 +117,7 @@ module SidekiqUniqueJobs
         end
       end
 
-      def active?(digest) # rubocop:disable Metrics/MethodLength
+      def active?(digest) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         Sidekiq.redis do |conn|
           procs = conn.sscan_each("processes").to_a
           return false if procs.empty?
@@ -132,7 +132,12 @@ module SidekiqUniqueJobs
             next unless workers.any?
 
             workers.each_pair do |_tid, job|
-              return true if load_json(job).dig(PAYLOAD, LOCK_DIGEST) == digest
+              next unless (item = safe_load_json(job))
+
+              payload = safe_load_json(item[PAYLOAD])
+
+              return true if payload[LOCK_DIGEST] == digest
+              return true if considered_active?(payload[CREATED_AT])
             end
           end
 
@@ -140,6 +145,10 @@ module SidekiqUniqueJobs
         end
       end
 
+      def considered_active?(time_f)
+        (Time.now - reaper_timeout).to_f < time_f
+      end
+
       #
       # Loops through all the redis queues and yields them one by one
       #

data/lib/sidekiq_unique_jobs/redis/entity.rb

@@ -48,10 +48,10 @@ module SidekiqUniqueJobs
       def exist?
         redis do |conn|
           value = conn.exists(key)
-          return true if value.is_a?(TrueClass)
-          return false if value.is_a?(FalseClass)
 
-          value.positive?
+          return value if boolean?(value)
+
+          value.to_i.positive?
         end
       end
 
@@ -95,6 +95,12 @@ module SidekiqUniqueJobs
       def count
         0
       end
+
+      private
+
+      def boolean?(value)
+        [TrueClass, FalseClass].any? { |klazz| value.is_a?(klazz) }
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/redis/sorted_set.rb

@@ -23,6 +23,23 @@ module SidekiqUniqueJobs
         entrys.each_with_object({}) { |pair, hash| hash[pair[0]] = pair[1] }
       end
 
+      #
+      # Adds a value to the sorted set
+      #
+      # @param [Array<Float, String>, String] the values to add
+      #
+      # @return [Boolean, Integer] <description>
+      #
+      def add(values)
+        redis do |conn|
+          if values.is_a?(Array)
+            conn.zadd(key, values)
+          else
+            conn.zadd(key, now_f, values)
+          end
+        end
+      end
+
       #
       # Return the zrak of the member
       #
@@ -45,6 +62,16 @@ module SidekiqUniqueJobs
         redis { |conn| conn.zscore(key, member) }
       end
 
+      #
+      # Clears the sorted set from all entries
+      #
+      #
+      # @return [Integer] number of entries removed
+      #
+      def clear
+        redis { |conn| conn.zremrangebyrank(key, 0, count) }
+      end
+
       #
       # Returns the count for this sorted set
       #

data/lib/sidekiq_unique_jobs/server.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # The unique sidekiq middleware for the server processor
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Server
+    DEATH_HANDLER ||= (lambda do |job, _ex|
+      return unless (digest = job["lock_digest"])
+
+      SidekiqUniqueJobs::Digests.new.delete_by_digest(digest)
+    end).freeze
+    #
+    # Configure the server middleware
+    #
+    #
+    # @return [Sidekiq] the sidekiq configuration
+    #
+    def self.configure(config)
+      config.on(:startup)  { start }
+      config.on(:shutdown) { stop }
+
+      return unless config.respond_to?(:death_handlers)
+
+      config.death_handlers << death_handler
+    end
+
+    def self.start
+      SidekiqUniqueJobs::UpdateVersion.call
+      SidekiqUniqueJobs::UpgradeLocks.call
+      SidekiqUniqueJobs::Orphans::Manager.start
+    end
+
+    def self.stop
+      SidekiqUniqueJobs::Orphans::Manager.stop
+    end
+
+    #
+    # A death handler for dead jobs
+    #
+    #
+    # @return [lambda]
+    #
+    def self.death_handler
+      DEATH_HANDLER
+    end
+  end
+end