sidekiq-unique-jobs 7.0.0.beta25 → 7.0.0

data/lib/sidekiq_unique_jobs/lock_ttl.rb

@@ -3,7 +3,7 @@
 module SidekiqUniqueJobs
   # Calculates timeout and expiration
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   class LockTTL
     # includes "SidekiqUniqueJobs::SidekiqWorkerMethods"
     # @!parse include SidekiqUniqueJobs::SidekiqWorkerMethods

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -3,7 +3,7 @@
 module SidekiqUniqueJobs
   # Lock manager class that handles all the various locks
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   class Locksmith # rubocop:disable Metrics/ClassLength
     # includes "SidekiqUniqueJobs::Connection"
     # @!parse include SidekiqUniqueJobs::Connection
@@ -87,7 +87,7 @@ module SidekiqUniqueJobs
     #
     def lock(&block)
       redis(redis_pool) do |conn|
-        return lock_async(conn, &block) if block_given?
+        return lock_async(conn, &block) if block
 
         lock_sync(conn) do
           return job_id

data/lib/sidekiq_unique_jobs/logging.rb

@@ -3,7 +3,7 @@
 module SidekiqUniqueJobs
   # Utility module for reducing the number of uses of logger.
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Logging
     def self.included(base)
       base.send(:extend, self)

data/lib/sidekiq_unique_jobs/logging/middleware_context.rb

@@ -4,13 +4,13 @@ module SidekiqUniqueJobs
   #
   # Provides the sidekiq middleware that makes the gem work
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   module Logging
     #
     # Context aware logging for Sidekiq Middlewares
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
     module Middleware
       include Logging

data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua

@@ -25,7 +25,7 @@ local redisversion = tostring(ARGV[5])
 --------  BEGIN delete_by_digest.lua --------
 local counter       = 0
 local redis_version = toversion(redisversion)
-local del_cmd   = "DEL"
+local del_cmd       = "DEL"
 
 log_debug("BEGIN delete_by_digest:", digest)
 

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -69,8 +69,7 @@ redis.call("LREM", queued, -1, job_id)
 log_debug("LREM", primed, 1, job_id)
 redis.call("LREM", primed, 1, job_id)
 
--- The Sidekiq client should only set pttl for until_expired
--- The Sidekiq server should set pttl for all other jobs
+-- The Sidekiq client sets pttl
 if pttl and pttl > 0 then
   log_debug("PEXPIRE", digest, pttl)
   redis.call("PEXPIRE", digest, pttl)

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -7,15 +7,16 @@ local retry_set    = KEYS[3]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
-local reaper_count  = tonumber(ARGV[1])
+local reaper_count = tonumber(ARGV[1])
+local threshold    = tonumber(ARGV[2])
 --------  END argv  ---------
 
 --------  BEGIN injected arguments --------
-local current_time = tonumber(ARGV[2])
-local debug_lua    = ARGV[3] == "true"
-local max_history  = tonumber(ARGV[4])
-local script_name  = ARGV[5] .. ".lua"
-local redisversion = ARGV[6]
+local current_time = tonumber(ARGV[3])
+local debug_lua    = ARGV[4] == "true"
+local max_history  = tonumber(ARGV[5])
+local script_name  = ARGV[6] .. ".lua"
+local redisversion = ARGV[7]
 ---------  END injected arguments ---------
 
 
@@ -65,7 +66,7 @@ repeat
     -- TODO: Add check for jobs checked out by process
     if found ~= true then
       log_debug("Searching for digest:", digest, "in process sets")
-      found = find_digest_in_process_set(digest)
+      found = find_digest_in_process_set(digest, threshold)
     end
 
     if found ~= true then

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua

@@ -1,4 +1,4 @@
-local function find_digest_in_process_set(digest)
+local function find_digest_in_process_set(digest, threshold)
   local process_cursor = 0
   local job_cursor     = 0
   local pattern        = "*" .. digest .. "*"
@@ -26,11 +26,18 @@ local function find_digest_in_process_set(digest)
         log_debug("No entries in:", workers_key)
       else
         for i = 1, #jobs, 2 do
-          if string.find(jobs[i +1], digest) then
+          local jobstr = jobs[i +1]
+          if string.find(jobstr, digest) then
             log_debug("Found digest", digest, "in:", workers_key)
             found = true
             break
           end
+
+          local job = cjson.decode(jobstr)
+          if job.created_at > threshold then
+            found = true
+            break
+          end
         end
       end
 

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -4,70 +4,13 @@ module SidekiqUniqueJobs
   #
   # Provides the sidekiq middleware that makes the gem work
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   module Middleware
     include SidekiqUniqueJobs::Logging::Middleware
     include SidekiqUniqueJobs::OptionsWithFallback
     include SidekiqUniqueJobs::JSON
 
-    #
-    # Configure both server and client
-    #
-    def self.configure
-      configure_server
-      configure_client
-    end
-
-    #
-    # Configures the Sidekiq server
-    #
-    def self.configure_server # rubocop:disable Metrics/MethodLength
-      Sidekiq.configure_server do |config|
-        config.client_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Client
-          end
-        end
-
-        config.server_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Server)
-            chain.insert_after Apartment::Sidekiq::Middleware::Server, SidekiqUniqueJobs::Middleware::Server
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Server
-          end
-        end
-
-        config.on(:startup) do
-          SidekiqUniqueJobs::UpdateVersion.call
-          SidekiqUniqueJobs::UpgradeLocks.call
-
-          SidekiqUniqueJobs::Orphans::Manager.start
-        end
-
-        config.on(:shutdown) do
-          SidekiqUniqueJobs::Orphans::Manager.stop
-        end
-      end
-    end
-
-    #
-    # Configures the Sidekiq client
-    #
-    def self.configure_client
-      Sidekiq.configure_client do |config|
-        config.client_middleware do |chain|
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
-          else
-            chain.add SidekiqUniqueJobs::Middleware::Client
-          end
-        end
-      end
-    end
-
     # The sidekiq job hash
     # @return [Hash] the Sidekiq job hash
     attr_reader :item

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
   module Middleware
     # The unique sidekiq middleware for the client push
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Client
       prepend SidekiqUniqueJobs::Middleware
 

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
   module Middleware
     # The unique sidekiq middleware for the server processor
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Server
       prepend SidekiqUniqueJobs::Middleware
 

data/lib/sidekiq_unique_jobs/normalizer.rb

@@ -3,7 +3,7 @@
 module SidekiqUniqueJobs
   # Normalizes hashes by dumping them to json and loading them from json
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Normalizer
     extend SidekiqUniqueJobs::JSON
 

data/lib/sidekiq_unique_jobs/on_conflict.rb

@@ -12,7 +12,7 @@ module SidekiqUniqueJobs
   #
   # Provides lock conflict resolutions
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   module OnConflict
     # A convenience method for using the configured strategies

data/lib/sidekiq_unique_jobs/on_conflict/log.rb

@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
   module OnConflict
     # Strategy to log information about conflict
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Log < OnConflict::Strategy
       include SidekiqUniqueJobs::Logging
 

data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb

@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
   module OnConflict
     # Default conflict strategy class that does nothing
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class NullStrategy < OnConflict::Strategy
       # Do nothing on conflict
       # @return [nil]

data/lib/sidekiq_unique_jobs/on_conflict/raise.rb

@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
   module OnConflict
     # Strategy to raise an error on conflict
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Raise < OnConflict::Strategy
       # Raise an error on conflict.
       #   This will cause Sidekiq to retry the job

data/lib/sidekiq_unique_jobs/on_conflict/reject.rb

@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
   module OnConflict
     # Strategy to send jobs to dead queue
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Reject < OnConflict::Strategy
       include SidekiqUniqueJobs::Timing
 

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -4,16 +4,16 @@ module SidekiqUniqueJobs
   module OnConflict
     # Strategy to replace the job on conflict
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Replace < OnConflict::Strategy
       #
       # @!attribute [r] queue
       #   @return [String] rthe sidekiq queue this job belongs to
       attr_reader :queue
       #
-      # @!attribute [r] unique_digest
+      # @!attribute [r] lock_digest
       #   @return [String] the unique digest to use for locking
-      attr_reader :unique_digest
+      attr_reader :lock_digest
 
       #
       # Initialize a new Replace strategy
@@ -22,8 +22,8 @@ module SidekiqUniqueJobs
       #
       def initialize(item, redis_pool = nil)
         super(item, redis_pool)
-        @queue         = item[QUEUE]
-        @unique_digest = item[LOCK_DIGEST]
+        @queue       = item[QUEUE]
+        @lock_digest = item[LOCK_DIGEST]
       end
 
       #
@@ -37,10 +37,11 @@ module SidekiqUniqueJobs
       def call(&block)
         return unless (deleted_job = delete_job_by_digest)
 
-        log_info("Deleting job: #{deleted_job}")
+        log_info("Deleted job: #{deleted_job}")
         if (del_count = delete_lock)
-          log_info("Deleted `#{del_count}` keys for #{unique_digest}")
+          log_info("Deleted `#{del_count}` keys for #{lock_digest}")
         end
+
         block&.call
       end
 
@@ -54,7 +55,7 @@ module SidekiqUniqueJobs
       def delete_job_by_digest
         call_script(:delete_job_by_digest,
                     keys: ["#{QUEUE}:#{queue}", SCHEDULE, RETRY],
-                    argv: [unique_digest])
+                    argv: [lock_digest])
       end
 
       #
@@ -64,7 +65,7 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of keys deleted
       #
       def delete_lock
-        digests.delete_by_digest(unique_digest)
+        digests.delete_by_digest(lock_digest)
       end
 
       #

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
   module OnConflict
     # Strategy to reschedule job on conflict
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Reschedule < OnConflict::Strategy
       include SidekiqUniqueJobs::SidekiqWorkerMethods
       include SidekiqUniqueJobs::Logging
@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
       def call
         if sidekiq_worker_class?
           log_info("Rescheduling #{item[LOCK_DIGEST]}")
-          worker_class&.perform_in(5, *item[ARGS])
+          worker_class.perform_in(5, *item[ARGS])
         else
           log_warn("Skip rescheduling of #{item[LOCK_DIGEST]} because #{worker_class} is not a Sidekiq::Worker")
         end

data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb

@@ -5,7 +5,7 @@ module SidekiqUniqueJobs
     # Abstract conflict strategy class
     #
     # @abstract
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Strategy
       include SidekiqUniqueJobs::JSON
       include SidekiqUniqueJobs::Logging

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -7,7 +7,7 @@ module SidekiqUniqueJobs
   #   1. item (required)
   #   2. options (can be nil)
   #   3. worker_class (required, can be anything)
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module OptionsWithFallback
     def self.included(base)
       base.send(:include, SidekiqUniqueJobs::SidekiqWorkerMethods)

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -7,7 +7,7 @@ module SidekiqUniqueJobs
     #
     # @note this is a much slower version of the lua script but does not crash redis
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
     class LuaReaper < Reaper
       #
@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
           :reap_orphans,
           conn,
           keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
-          argv: [reaper_count],
+          argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end
     end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -5,12 +5,13 @@ module SidekiqUniqueJobs
     #
     # Manages the orphan reaper
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
     module Manager
       module_function
 
       DRIFT_FACTOR = 0.02
+      REAPERS      = [:ruby, :lua].freeze
 
       include SidekiqUniqueJobs::Connection
       include SidekiqUniqueJobs::Logging
@@ -22,8 +23,8 @@ module SidekiqUniqueJobs
       # @return [Concurrent::TimerTask] the task that was started
       #
       def start # rubocop:disable
-        return if registered?
         return if disabled?
+        return if registered?
 
         with_logging_context do
           register_reaper_process
@@ -41,6 +42,9 @@ module SidekiqUniqueJobs
       # @return [Boolean]
       #
       def stop
+        return if disabled?
+        return if unregistered?
+
         with_logging_context do
           log_info("Stopping Reaper")
           unregister_reaper_process
@@ -55,7 +59,12 @@ module SidekiqUniqueJobs
       # @return [<type>] <description>
       #
       def task
-        @task ||= Concurrent::TimerTask.new(timer_task_options) do
+        @task ||= Concurrent::TimerTask.new(timer_task_options, &task_body)
+      end
+
+      # @private
+      def task_body
+        @task_body ||= lambda do
           with_logging_context do
             redis do |conn|
               refresh_reaper_mutex
@@ -114,7 +123,7 @@ module SidekiqUniqueJobs
       end
 
       #
-      # Checks if a reaper is already registered
+      # Checks if a reaper is registered
       #
       #
       # @return [true, false]
@@ -125,8 +134,35 @@ module SidekiqUniqueJobs
         end
       end
 
+      #
+      # Checks if that reapers are not registerd
+      #
+      # @see registered?
+      #
+      # @return [true, false]
+      #
+      def unregistered?
+        !registered?
+      end
+
+      #
+      # Checks if reaping is disabled
+      #
+      # @see enabled?
+      #
+      # @return [true, false]
+      #
       def disabled?
-        reaper == :none
+        !enabled?
+      end
+
+      #
+      # Checks if reaping is enabled
+      #
+      # @return [true, false]
+      #
+      def enabled?
+        REAPERS.include?(reaper)
       end
 
       #