sidekiq-unique-jobs 7.1.25 → 7.1.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a5c0d9d30f11e61d3cc7599343951d242c5ccba2c93ee2a46138eab1037db81
-  data.tar.gz: a0721afd3b1a4e98655f9d1aba34e609d9ddd71bf6fd02d2b6f35bfbf4f9bece
+  metadata.gz: f6c5acd562db0da327940682a24bf4c28793a61ffb9904c3f480352253676b4e
+  data.tar.gz: 1673f51ceb70a1e6f37e60a411a93fe6ed9a05091a6e044b38283f2dbc93bbaf
 SHA512:
-  metadata.gz: 7751e266ae6df52925ba2b0b3fbc72e7fff5e2e979ae00dcc8fb7af433aac22627901a650cdd53f64479af440e68a814ad3d67f446509e4aef4351e6eed71da0
-  data.tar.gz: 210d306f47c106bab3ae6e109c24d728baf5a81cc6caa3ee309a7c96f547c5cd8503e99cb7f814671c1feccfdaa48ea992581c9f978bb2bfd662c51946e13ffd
+  metadata.gz: 90dc8a981cd53213457bec790de3460823317666693ba56d5bc8d4f890a8a376e125ba5034f438109f1e883a847c09341634aa0bc8d509ed0d21534f53f49b4b
+  data.tar.gz: b4d4e314415706035afb5b3b078be70acb762134aa75e89be87d8e76bed0cdafec1e27ec9d303cae596b41ad5e504dccc01dd989256cd4272e29ebbb47e8f4cf

data/CHANGELOG.md

@@ -1,5 +1,44 @@
 # Changelog
 
+## [Unreleased](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/HEAD)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.26...HEAD)
+
+**Closed issues:**
+
+- Memory bloat / dangling keys / reaper not cleaning orphans [\#637](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/637)
+
+## [v7.1.26](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.26) (2022-07-28)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.25...v7.1.26)
+
+**Implemented enhancements:**
+
+- Fix\(until\_expired\): Fix test and implementation [\#725](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/725) ([mhenrixon](https://github.com/mhenrixon))
+
+**Fixed bugs:**
+
+- Fix\(until\_and\_while\_executing\): Improve timeouts slightly [\#728](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/728) ([mhenrixon](https://github.com/mhenrixon))
+- Fix\(unlock\): Delete primed keys on last entry [\#726](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/726) ([mhenrixon](https://github.com/mhenrixon))
+
+**Merged pull requests:**
+
+- Ensure batch delete removes expiring locks [\#724](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/724) ([francesmcmullin](https://github.com/francesmcmullin))
+- Chore: Update dependencies [\#722](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/722) ([mhenrixon](https://github.com/mhenrixon))
+- Move until\_expired digests to separate zset [\#721](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/721) ([francesmcmullin](https://github.com/francesmcmullin))
+- Avoid skipping ranges when looping through queues [\#720](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/720) ([francesmcmullin](https://github.com/francesmcmullin))
+- Bump actions/checkout from 2 to 3 [\#718](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/718) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Add Dependabot for GitHub Actions [\#717](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/717) ([petergoldstein](https://github.com/petergoldstein))
+- Fix Sidekiq::Worker.clear\_all override not being applied [\#714](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/714) ([dsander](https://github.com/dsander))
+
+## [v7.1.25](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.25) (2022-06-13)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.24...v7.1.25)
+
+**Fixed bugs:**
+
+- Fix: Include the correct middleware [\#716](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/716) ([mhenrixon](https://github.com/mhenrixon))
+
 ## [v7.1.24](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.24) (2022-06-09)
 
 [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.23...v7.1.24)

data/README.md

@@ -31,7 +31,7 @@ Want to show me some ❤️ for the hard work I do on this gem? You can use the
   - [raise](#raise)
   - [reject](#reject)
   - [replace](#replace)
-  - [Reschedule](#reschedule)
+  - [reschedule](#reschedule)
   - [Custom Strategies](#custom-strategies)
   - [3 Cleanup Dead Locks](#3-cleanup-dead-locks)
 - [Debugging](#debugging)
@@ -610,6 +610,7 @@ This has been probably the most confusing part of this gem. People get really co
 ```ruby
 SidekiqUniqueJobs.configure do |config|
   config.enabled = !Rails.env.test?
+  config.logger_enabled = !Rails.env.test?
 end
 ```
 
@@ -736,6 +737,7 @@ Configure SidekiqUniqueJobs in an initializer or the sidekiq initializer on appl
 ```ruby
 SidekiqUniqueJobs.configure do |config|
   config.logger = Sidekiq.logger # default, change at your own discretion
+  config.logger_enabled  = true # default, disable for test environments
   config.debug_lua       = false # Turn on when debugging
   config.lock_info       = false # Turn on when debugging
   config.lock_ttl        = 600   # Expire locks after 10 minutes

data/lib/sidekiq_unique_jobs/batch_delete.rb

@@ -91,6 +91,7 @@ module SidekiqUniqueJobs
           chunk.each do |digest|
             del_digest(pipeline, digest)
             pipeline.zrem(SidekiqUniqueJobs::DIGESTS, digest)
+            pipeline.zrem(SidekiqUniqueJobs::EXPIRING_DIGESTS, digest)
             @count += 1
           end
         end

data/lib/sidekiq_unique_jobs/cli.rb

@@ -20,9 +20,11 @@ module SidekiqUniqueJobs
     option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
     # :nodoc:
     def list(pattern = "*")
-      entries = digests.entries(pattern: pattern, count: options[:count])
-      say "Found #{entries.size} digests matching '#{pattern}':"
-      print_in_columns(entries.sort) if entries.any?
+      max_count = options[:count]
+      say "Searching for regular digests"
+      list_entries(digests.entries(pattern: pattern, count: max_count), pattern)
+      say "Searching for expiring digests"
+      list_entries(expiring_digests.entries(pattern: pattern, count: max_count), pattern)
     end
 
     desc "del PATTERN", "deletes unique digests from redis by pattern"
@@ -32,11 +34,9 @@ module SidekiqUniqueJobs
     def del(pattern)
       max_count = options[:count]
       if options[:dry_run]
-        result = digests.entries(pattern: pattern, count: max_count)
-        say "Would delete #{result.size} digests matching '#{pattern}'"
+        count_entries_for_del(max_count, pattern)
       else
-        deleted_count = digests.delete_by_pattern(pattern, count: max_count)
-        say "Deleted #{deleted_count} digests matching '#{pattern}'"
+        del_entries(max_count, pattern)
       end
     end
 
@@ -51,12 +51,17 @@ module SidekiqUniqueJobs
       console_class.start
     end
 
-    no_commands do
+    no_commands do # rubocop:disable Metrics/BlockLength
       # :nodoc:
       def digests
         @digests ||= SidekiqUniqueJobs::Digests.new
       end
 
+      # :nodoc:
+      def expiring_digests
+        @expiring_digests ||= SidekiqUniqueJobs::ExpiringDigests.new
+      end
+
       # :nodoc:
       def console_class
         require "pry"
@@ -65,6 +70,26 @@ module SidekiqUniqueJobs
         require "irb"
         IRB
       end
+
+      # :nodoc:
+      def list_entries(entries, pattern)
+        say "Found #{entries.size} digests matching '#{pattern}':"
+        print_in_columns(entries.sort) if entries.any?
+      end
+
+      # :nodoc:
+      def count_entries_for_del(max_count, pattern)
+        count = digests.entries(pattern: pattern, count: max_count).size +
+                expiring_digests.entries(pattern: pattern, count: max_count).size
+        say "Would delete #{count} digests matching '#{pattern}'"
+      end
+
+      # :nodoc:
+      def del_entries(max_count, pattern)
+        deleted_count = digests.delete_by_pattern(pattern, count: max_count).to_i +
+                        expiring_digests.delete_by_pattern(pattern, count: max_count).to_i
+        say "Deleted #{deleted_count} digests matching '#{pattern}'"
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/config.rb

@@ -8,6 +8,7 @@ module SidekiqUniqueJobs
                                                    :enabled,
                                                    :lock_prefix,
                                                    :logger,
+                                                   :logger_enabled,
                                                    :locks,
                                                    :strategies,
                                                    :debug_lua,
@@ -91,6 +92,9 @@ module SidekiqUniqueJobs
     # @return [nil]
     LOCK_TTL              = nil
     #
+    # @return [true,false] by default false (don't disable logger)
+    LOGGER_ENABLED        = true
+    #
     # @return [true] by default the gem is enabled
     ENABLED               = true
     #
@@ -180,6 +184,7 @@ module SidekiqUniqueJobs
         ENABLED,
         PREFIX,
         Sidekiq.logger,
+        LOGGER_ENABLED,
         LOCKS,
         STRATEGIES,
         DEBUG_LUA,

data/lib/sidekiq_unique_jobs/constants.rb

@@ -14,6 +14,7 @@ module SidekiqUniqueJobs
   CREATED_AT            = "created_at"
   DEAD_VERSION          = "uniquejobs:dead"
   DIGESTS               = "uniquejobs:digests"
+  EXPIRING_DIGESTS      = "uniquejobs:expiring_digests"
   ERRORS                = "errors"
   JID                   = "jid"
   LIMIT                 = "limit"

data/lib/sidekiq_unique_jobs/digests.rb

@@ -14,8 +14,8 @@ module SidekiqUniqueJobs
     # @return [String] the default pattern to use for matching
     SCAN_PATTERN  = "*"
 
-    def initialize
-      super(DIGESTS)
+    def initialize(digests_key = DIGESTS)
+      super(digests_key)
     end
 
     #

data/lib/sidekiq_unique_jobs/expiring_digests.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class ExpiringDigests provides access to the expiring digests used by until_expired locks
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class ExpiringDigests < Digests
+    def initialize
+      super(EXPIRING_DIGESTS)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/key.rb

@@ -33,6 +33,10 @@ module SidekiqUniqueJobs
     # @!attribute [r] digests
     #   @return [String] the zset with locked digests
     attr_reader :digests
+    #
+    # @!attribute [r] expiring_digests
+    #   @return [String] the zset with locked expiring_digests
+    attr_reader :expiring_digests
 
     #
     # Initialize a new Key
@@ -40,13 +44,14 @@ module SidekiqUniqueJobs
     # @param [String] digest the digest to use as key
     #
     def initialize(digest)
-      @digest    = digest
-      @queued    = suffixed_key("QUEUED")
-      @primed    = suffixed_key("PRIMED")
-      @locked    = suffixed_key("LOCKED")
-      @info      = suffixed_key("INFO")
-      @changelog = CHANGELOGS
-      @digests   = DIGESTS
+      @digest           = digest
+      @queued           = suffixed_key("QUEUED")
+      @primed           = suffixed_key("PRIMED")
+      @locked           = suffixed_key("LOCKED")
+      @info             = suffixed_key("INFO")
+      @changelog        = CHANGELOGS
+      @digests          = DIGESTS
+      @expiring_digests = EXPIRING_DIGESTS
     end
 
     #
@@ -81,7 +86,7 @@ module SidekiqUniqueJobs
     # @return [Array] an ordered array with all keys
     #
     def to_a
-      [digest, queued, primed, locked, info, changelog, digests]
+      [digest, queued, primed, locked, info, changelog, digests, expiring_digests]
     end
 
     private

data/lib/sidekiq_unique_jobs/lock.rb

@@ -66,7 +66,7 @@ module SidekiqUniqueJobs
           pipeline.set(key.digest, job_id)
           pipeline.hset(key.locked, job_id, now_f)
           info.set(lock_info, pipeline)
-          pipeline.zadd(key.digests, now_f, key.digest)
+          add_digest_to_set(pipeline, lock_info)
           pipeline.zadd(key.changelog, now_f, changelog_json(job_id, "queue.lua", "Queued"))
           pipeline.zadd(key.changelog, now_f, changelog_json(job_id, "lock.lua", "Locked"))
         end
@@ -321,5 +321,22 @@ module SidekiqUniqueJobs
         time: now_f,
       )
     end
+
+    #
+    # Add the digest to the correct sorted set
+    #
+    # @param [Object] pipeline a redis pipeline object for issue commands
+    # @param [Hash] lock_info the lock info relevant to the digest
+    #
+    # @return [nil]
+    #
+    def add_digest_to_set(pipeline, lock_info)
+      digest_string = key.digest
+      if lock_info["lock"] == :until_expired
+        pipeline.zadd(key.expiring_digests, now_f + lock_info["ttl"], digest_string)
+      else
+        pipeline.zadd(key.digests, now_f, digest_string)
+      end
+    end
   end
 end

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -32,6 +32,7 @@ module SidekiqUniqueJobs
     #
     # @return [Float] used to take into consideration the inaccuracy of redis timestamps
     CLOCK_DRIFT_FACTOR = 0.01
+    NETWORK_FACTOR = 0.04
 
     #
     # @!attribute [r] key
@@ -184,6 +185,7 @@ module SidekiqUniqueJobs
     #
     # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
     # @param [Method] primed_method reference to the method to use for getting a primed token
+    # @param [nil, Integer, Float] time to wait before timeout
     #
     # @yieldparam [string] job_id the sidekiq JID
     # @yieldreturn [void] whatever the calling block returns
@@ -239,9 +241,18 @@ module SidekiqUniqueJobs
     # @return [Object] whatever the block returns when lock was acquired
     #
     def primed_async(conn, wait = nil, &block)
+      timeout = (wait || config.timeout).to_i
+      timeout = 1 if timeout.zero?
+
+      brpoplpush_timeout = timeout
+      concurrent_timeout = add_drift(timeout)
+
+      reflect(:debug, :timeouts, item,
+              timeouts: { brpoplpush_timeout: brpoplpush_timeout, concurrent_timeout: concurrent_timeout })
+
       primed_jid = Concurrent::Promises
-                   .future(conn) { |red_con| pop_queued(red_con, wait) }
-                   .value(add_drift(wait || config.timeout))
+                   .future(conn) { |red_con| pop_queued(red_con, timeout) }
+                   .value
 
       handle_primed(primed_jid, &block)
     end
@@ -273,7 +284,7 @@ module SidekiqUniqueJobs
     #
     # @return [String] a previously enqueued token (now taken off the queue)
     #
-    def pop_queued(conn, wait = nil)
+    def pop_queued(conn, wait = 1)
       wait ||= config.timeout if config.wait_for_lock?
 
       if wait.nil?
@@ -287,10 +298,15 @@ module SidekiqUniqueJobs
     # @api private
     #
     def brpoplpush(conn, wait)
+      # passing timeout 0 to brpoplpush causes it to block indefinitely
       raise InvalidArgument, "wait must be an integer" unless wait.is_a?(Integer)
+      return conn.brpoplpush(key.queued, key.primed, wait) if conn.class.to_s == "Redis::Namespace"
 
-      # passing timeout 0 to brpoplpush causes it to block indefinitely
-      conn.brpoplpush(key.queued, key.primed, timeout: wait)
+      if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && conn.respond_to?(:blmove)
+        conn.blmove(key.queued, key.primed, "RIGHT", "LEFT", timeout: wait)
+      else
+        conn.brpoplpush(key.queued, key.primed, timeout: wait)
+      end
     end
 
     #
@@ -359,5 +375,9 @@ module SidekiqUniqueJobs
         TIME => now_f,
       )
     end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
   end
 end

data/lib/sidekiq_unique_jobs/logging.rb

@@ -30,6 +30,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_debug(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.debug(message, &block)
       nil
@@ -45,6 +47,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_info(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.info(message, &block)
       nil
@@ -60,6 +64,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_warn(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.warn(message, &block)
       nil
@@ -75,6 +81,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_error(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.error(message, &block)
       nil
@@ -90,6 +98,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_fatal(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.fatal(message, &block)
 
@@ -218,5 +228,9 @@ module SidekiqUniqueJobs
 
       yield
     end
+
+    def logging?
+      SidekiqUniqueJobs.logging?
+    end
   end
 end

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -1,11 +1,12 @@
 -------- BEGIN keys ---------
-local digest    = KEYS[1]
-local queued    = KEYS[2]
-local primed    = KEYS[3]
-local locked    = KEYS[4]
-local info      = KEYS[5]
-local changelog = KEYS[6]
-local digests   = KEYS[7]
+local digest           = KEYS[1]
+local queued           = KEYS[2]
+local primed           = KEYS[3]
+local locked           = KEYS[4]
+local info             = KEYS[5]
+local changelog        = KEYS[6]
+local digests          = KEYS[7]
+local expiring_digests = KEYS[8]
 -------- END keys ---------
 
 
@@ -57,8 +58,13 @@ if limit_exceeded then
   return nil
 end
 
-log_debug("ZADD", digests, current_time, digest)
-redis.call("ZADD", digests, current_time, digest)
+if lock_type == "until_expired" and pttl and pttl > 0 then
+  log_debug("ZADD", expiring_digests, current_time + pttl, digest)
+  redis.call("ZADD", expiring_digests, current_time + pttl, digest)
+else
+  log_debug("ZADD", digests, current_time, digest)
+  redis.call("ZADD", digests, current_time, digest)
+end
 
 log_debug("HSET", locked, job_id, current_time)
 redis.call("HSET", locked, job_id, current_time)

data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua

@@ -0,0 +1,92 @@
+-------- BEGIN keys ---------
+local digest           = KEYS[1]
+local queued           = KEYS[2]
+local primed           = KEYS[3]
+local locked           = KEYS[4]
+local info             = KEYS[5]
+local changelog        = KEYS[6]
+local digests          = KEYS[7]
+local expiring_digests = KEYS[8]
+-------- END keys ---------
+
+
+-------- BEGIN lock arguments ---------
+local job_id       = ARGV[1]
+local pttl         = tonumber(ARGV[2])
+local lock_type    = ARGV[3]
+local limit        = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+---------  BEGIN lock.lua ---------
+log_debug("BEGIN lock digest:", digest, "job_id:", job_id)
+
+if redis.call("HEXISTS", locked, job_id) == 1 then
+  log_debug(locked, "already locked with job_id:", job_id)
+  log("Duplicate")
+
+  log_debug("LREM", queued, -1, job_id)
+  redis.call("LREM", queued, -1, job_id)
+
+  log_debug("LREM", primed, 1, job_id)
+  redis.call("LREM", primed, 1, job_id)
+
+  return job_id
+end
+
+local locked_count   = redis.call("HLEN", locked)
+local within_limit   = limit > locked_count
+local limit_exceeded = not within_limit
+
+if limit_exceeded then
+  log_debug("Limit exceeded:", digest, "(",  locked_count, "of", limit, ")")
+  log("Limited")
+  return nil
+end
+
+log_debug("ZADD", expiring_digests, current_time + pttl, digest)
+redis.call("ZADD", expiring_digests, current_time + pttl, digest)
+
+log_debug("HSET", locked, job_id, current_time)
+redis.call("HSET", locked, job_id, current_time)
+
+log_debug("LREM", queued, -1, job_id)
+redis.call("LREM", queued, -1, job_id)
+
+log_debug("LREM", primed, 1, job_id)
+redis.call("LREM", primed, 1, job_id)
+
+-- The Sidekiq client sets pttl
+log_debug("PEXPIRE", digest, pttl)
+redis.call("PEXPIRE", digest, pttl)
+
+log_debug("PEXPIRE", locked, pttl)
+redis.call("PEXPIRE", locked, pttl)
+
+log_debug("PEXPIRE", info, pttl)
+redis.call("PEXPIRE", info, pttl)
+
+log_debug("PEXPIRE", queued, 1000)
+redis.call("PEXPIRE", queued, 1000)
+
+log_debug("PEXPIRE", primed, 1000)
+redis.call("PEXPIRE", primed, 1000)
+
+log("Locked")
+log_debug("END lock digest:", digest, "job_id:", job_id)
+return job_id
+----------  END lock.lua  ----------

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -1,9 +1,10 @@
 redis.replicate_commands()
 
 -------- BEGIN keys ---------
-local digests_set  = KEYS[1]
-local schedule_set = KEYS[2]
-local retry_set    = KEYS[3]
+local digests_set          = KEYS[1]
+local expiring_digests_set = KEYS[2]
+local schedule_set         = KEYS[3]
+local retry_set            = KEYS[4]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
@@ -90,5 +91,32 @@ repeat
   index = index + per
 until index >= total or del_count >= reaper_count
 
+if del_count < reaper_count then
+  index = 0
+  total = redis.call("ZCOUNT", expiring_digests_set, 0, current_time)
+  repeat
+    local digests = redis.call("ZRANGEBYSCORE", expiring_digests_set, 0, current_time, "LIMIT", index, index + per -1)
+
+    for _, digest in pairs(digests) do
+      local queued     = digest .. ":QUEUED"
+      local primed     = digest .. ":PRIMED"
+      local locked     = digest .. ":LOCKED"
+      local info       = digest .. ":INFO"
+      local run_digest = digest .. ":RUN"
+      local run_queued = digest .. ":RUN:QUEUED"
+      local run_primed = digest .. ":RUN:PRIMED"
+      local run_locked = digest .. ":RUN:LOCKED"
+      local run_info   = digest .. ":RUN:INFO"
+
+      redis.call(del_cmd, digest, queued, primed, locked, info, run_digest, run_queued, run_primed, run_locked, run_info)
+
+      redis.call("ZREM", expiring_digests_set, digest)
+      del_count = del_count + 1
+    end
+
+    index = index + per
+  until index >= total or del_count >= reaper_count
+end
+
 log_debug("END")
 return del_count

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -85,6 +85,11 @@ if locked_count and locked_count < 1 then
   redis.call(del_cmd, locked)
 end
 
+if redis.call("LLEN", primed) == 0 then
+  log_debug(del_cmd, primed)
+  redis.call(del_cmd, primed)
+end
+
 if limit and limit <= 1 and locked_count and locked_count <= 1 then
   log_debug("ZREM", digests, digest)
   redis.call("ZREM", digests, digest)

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -20,7 +20,7 @@ module SidekiqUniqueJobs
         call_script(
           :reap_orphans,
           conn,
-          keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
+          keys: [DIGESTS, EXPIRING_DIGESTS, SCHEDULE, RETRY, PROCESSES],
           argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -34,9 +34,15 @@ module SidekiqUniqueJobs
 
       #
       # @!attribute [r] start_time
-      #   @return [Integer] The timestamp this execution started represented as integer
+      #   @return [Integer] The timestamp this execution started represented as Time (used for locks)
       attr_reader :start_time
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The clock stamp this execution started represented as integer
+      #      (used for redis compatibility as it is more accurate than time)
+      attr_reader :start_source
+
       #
       # @!attribute [r] timeout_ms
       #   @return [Integer] The allowed ms before timeout
@@ -49,11 +55,12 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests    = SidekiqUniqueJobs::Digests.new
-        @scheduled  = Redis::SortedSet.new(SCHEDULE)
-        @retried    = Redis::SortedSet.new(RETRY)
-        @start_time = time_source.call
-        @timeout_ms = SidekiqUniqueJobs.config.reaper_timeout * 1000
+        @digests      = SidekiqUniqueJobs::Digests.new
+        @scheduled    = Redis::SortedSet.new(SCHEDULE)
+        @retried      = Redis::SortedSet.new(RETRY)
+        @start_time   = Time.now
+        @start_source = time_source.call
+        @timeout_ms   = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -65,9 +72,20 @@ module SidekiqUniqueJobs
       def call
         return if queues_very_full?
 
+        BatchDelete.call(expired_digests, conn)
         BatchDelete.call(orphans, conn)
       end
 
+      def expired_digests
+        max_score = (start_time - reaper_timeout).to_f
+
+        if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && VersionCheck.satisfied?(::Redis::VERSION, ">= 4.6.0")
+          conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
+        else
+          conn.zrangebyscore(EXPIRING_DIGESTS, 0, max_score)
+        end
+      end
+
       #
       # Find orphaned digests
       #
@@ -104,7 +122,7 @@ module SidekiqUniqueJobs
       end
 
       def elapsed_ms
-        time_source.call - start_time
+        time_source.call - start_source
       end
 
       #
@@ -229,6 +247,7 @@ module SidekiqUniqueJobs
 
         loop do
           range_start = (page * page_size) - deleted_size
+
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1
@@ -238,6 +257,9 @@ module SidekiqUniqueJobs
           entries.each(&block)
 
           deleted_size = initial_size - conn.llen(queue_key)
+
+          # The queue is growing, not shrinking, just keep looping
+          deleted_size = 0 if deleted_size.negative?
         end
       end
 

data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb

@@ -71,6 +71,16 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
     config.logger = other
   end
 
+  #
+  # Check if logging is enabled
+  #
+  #
+  # @return [true, false]
+  #
+  def logging?
+    config.logger_enabled
+  end
+
   #
   # Temporarily use another configuration and reset to the old config after yielding
   #

data/lib/sidekiq_unique_jobs/testing.rb

@@ -90,15 +90,6 @@ module Sidekiq
         super(options)
       end
 
-      #
-      # Clears all jobs for this worker and removes all locks
-      #
-      def clear_all
-        super
-
-        SidekiqUniqueJobs::Digests.new.delete_by_pattern("*", count: 10_000)
-      end
-
       #
       # Prepends deletion of locks to clear
       #
@@ -124,5 +115,21 @@ module Sidekiq
     module ClassMethods
       prepend Overrides::ClassMethods
     end
+
+    #
+    # Prepends singleton methods to Sidekiq::Worker
+    #
+    module SignletonOverrides
+      #
+      # Clears all jobs for this worker and removes all locks
+      #
+      def clear_all
+        super
+
+        SidekiqUniqueJobs::Digests.new.delete_by_pattern("*", count: 10_000)
+      end
+    end
+
+    singleton_class.prepend SignletonOverrides
   end
 end

data/lib/sidekiq_unique_jobs/version.rb

@@ -3,5 +3,5 @@
 module SidekiqUniqueJobs
   #
   # @return [String] the current SidekiqUniqueJobs version
-  VERSION = "7.1.25"
+  VERSION = "7.1.27"
 end

data/lib/sidekiq_unique_jobs/web/helpers.rb

@@ -51,6 +51,16 @@ module SidekiqUniqueJobs
         @digests ||= SidekiqUniqueJobs::Digests.new
       end
 
+      #
+      # The collection of digests
+      #
+      #
+      # @return [SidekiqUniqueJobs::ExpiringDigests] the sorted set with expiring digests
+      #
+      def expiring_digests
+        @expiring_digests ||= SidekiqUniqueJobs::ExpiringDigests.new
+      end
+
       #
       # The collection of changelog entries
       #

data/lib/sidekiq_unique_jobs/web.rb

@@ -8,7 +8,7 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Web
-    def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       app.helpers do
         include Web::Helpers
       end
@@ -49,8 +49,25 @@ module SidekiqUniqueJobs
         erb(unique_template(:locks))
       end
 
+      app.get "/expiring_locks" do
+        @filter         = params[:filter] || "*"
+        @filter         = "*" if @filter == ""
+        @count          = (params[:count] || 100).to_i
+        @current_cursor = params[:cursor]
+        @prev_cursor    = params[:prev_cursor]
+
+        @total_size, @next_cursor, @locks = expiring_digests.page(
+          cursor: @current_cursor,
+          pattern: @filter,
+          page_size: @count,
+        )
+
+        erb(unique_template(:locks))
+      end
+
       app.get "/locks/delete_all" do
         digests.delete_by_pattern("*", count: digests.count)
+        expiring_digests.delete_by_pattern("*", count: digests.count)
         redirect_to :locks
       end
 
@@ -63,6 +80,7 @@ module SidekiqUniqueJobs
 
       app.get "/locks/:digest/delete" do
         digests.delete_by_digest(params[:digest])
+        expiring_digests.delete_by_digest(params[:digest])
         redirect_to :locks
       end
 
@@ -82,8 +100,9 @@ begin
   require "sidekiq/web" unless defined?(Sidekiq::Web)
 
   Sidekiq::Web.register(SidekiqUniqueJobs::Web)
-  Sidekiq::Web.tabs["Locks"]      = "locks"
-  Sidekiq::Web.tabs["Changelogs"] = "changelogs"
+  Sidekiq::Web.tabs["Locks"]          = "locks"
+  Sidekiq::Web.tabs["Expiring Locks"] = "expiring_locks"
+  Sidekiq::Web.tabs["Changelogs"]     = "changelogs"
   Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
 rescue NameError, LoadError => ex
   SidekiqUniqueJobs.logger.error(ex)

data/lib/sidekiq_unique_jobs.rb

@@ -72,6 +72,7 @@ require "sidekiq_unique_jobs/sidekiq_unique_ext"
 require "sidekiq_unique_jobs/on_conflict"
 require "sidekiq_unique_jobs/changelog"
 require "sidekiq_unique_jobs/digests"
+require "sidekiq_unique_jobs/expiring_digests"
 
 require "sidekiq_unique_jobs/config"
 require "sidekiq_unique_jobs/sidekiq_unique_jobs"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 7.1.25
+  version: 7.1.27
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-14 00:00:00.000000000 Z
+date: 2022-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: brpoplpush-redis_script
@@ -116,6 +116,7 @@ files:
 - lib/sidekiq_unique_jobs/deprecation.rb
 - lib/sidekiq_unique_jobs/digests.rb
 - lib/sidekiq_unique_jobs/exceptions.rb
+- lib/sidekiq_unique_jobs/expiring_digests.rb
 - lib/sidekiq_unique_jobs/job.rb
 - lib/sidekiq_unique_jobs/json.rb
 - lib/sidekiq_unique_jobs/key.rb
@@ -144,6 +145,7 @@ files:
 - lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua
 - lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua
 - lib/sidekiq_unique_jobs/lua/lock.lua
+- lib/sidekiq_unique_jobs/lua/lock_until_expired.lua
 - lib/sidekiq_unique_jobs/lua/locked.lua
 - lib/sidekiq_unique_jobs/lua/queue.lua
 - lib/sidekiq_unique_jobs/lua/reap_orphans.lua