sidekiq-unique-jobs 7.1.26 → 7.1.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 501cb34e3378a86e0046733d3b948dd61ccf3faafc876433d2e1bf19125122b5
-  data.tar.gz: 628a44c9828484baf678b88437f76f056b4c31e549e68a853441280de98b41b4
+  metadata.gz: f6c5acd562db0da327940682a24bf4c28793a61ffb9904c3f480352253676b4e
+  data.tar.gz: 1673f51ceb70a1e6f37e60a411a93fe6ed9a05091a6e044b38283f2dbc93bbaf
 SHA512:
-  metadata.gz: 2b3b98d114a3f328eb63a99409d6a938897f4d96cc6bdc1bc048c02452f09e50afe42b6779d1f6831bbe269abbc7e896d4d47c1c9b3710dea472c34d66c283d6
-  data.tar.gz: 7a12231a8b638821fd989e5b23998f29df4e401eda3308a4e3d5a496b3a8e982af6056dae9275aca97ef76a0cec1353dc8c200a916d3db7cc46b4fa9e9b3325a
+  metadata.gz: 90dc8a981cd53213457bec790de3460823317666693ba56d5bc8d4f890a8a376e125ba5034f438109f1e883a847c09341634aa0bc8d509ed0d21534f53f49b4b
+  data.tar.gz: b4d4e314415706035afb5b3b078be70acb762134aa75e89be87d8e76bed0cdafec1e27ec9d303cae596b41ad5e504dccc01dd989256cd4272e29ebbb47e8f4cf

data/CHANGELOG.md

@@ -1,5 +1,36 @@
 # Changelog
 
+## [Unreleased](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/HEAD)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.26...HEAD)
+
+**Closed issues:**
+
+- Memory bloat / dangling keys / reaper not cleaning orphans [\#637](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/637)
+
+## [v7.1.26](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.26) (2022-07-28)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.25...v7.1.26)
+
+**Implemented enhancements:**
+
+- Fix\(until\_expired\): Fix test and implementation [\#725](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/725) ([mhenrixon](https://github.com/mhenrixon))
+
+**Fixed bugs:**
+
+- Fix\(until\_and\_while\_executing\): Improve timeouts slightly [\#728](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/728) ([mhenrixon](https://github.com/mhenrixon))
+- Fix\(unlock\): Delete primed keys on last entry [\#726](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/726) ([mhenrixon](https://github.com/mhenrixon))
+
+**Merged pull requests:**
+
+- Ensure batch delete removes expiring locks [\#724](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/724) ([francesmcmullin](https://github.com/francesmcmullin))
+- Chore: Update dependencies [\#722](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/722) ([mhenrixon](https://github.com/mhenrixon))
+- Move until\_expired digests to separate zset [\#721](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/721) ([francesmcmullin](https://github.com/francesmcmullin))
+- Avoid skipping ranges when looping through queues [\#720](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/720) ([francesmcmullin](https://github.com/francesmcmullin))
+- Bump actions/checkout from 2 to 3 [\#718](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/718) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Add Dependabot for GitHub Actions [\#717](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/717) ([petergoldstein](https://github.com/petergoldstein))
+- Fix Sidekiq::Worker.clear\_all override not being applied [\#714](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/714) ([dsander](https://github.com/dsander))
+
 ## [v7.1.25](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.25) (2022-06-13)
 
 [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.24...v7.1.25)

data/README.md

@@ -31,7 +31,7 @@ Want to show me some ❤️ for the hard work I do on this gem? You can use the
   - [raise](#raise)
   - [reject](#reject)
   - [replace](#replace)
-  - [Reschedule](#reschedule)
+  - [reschedule](#reschedule)
   - [Custom Strategies](#custom-strategies)
   - [3 Cleanup Dead Locks](#3-cleanup-dead-locks)
 - [Debugging](#debugging)
@@ -610,6 +610,7 @@ This has been probably the most confusing part of this gem. People get really co
 ```ruby
 SidekiqUniqueJobs.configure do |config|
   config.enabled = !Rails.env.test?
+  config.logger_enabled = !Rails.env.test?
 end
 ```
 
@@ -736,6 +737,7 @@ Configure SidekiqUniqueJobs in an initializer or the sidekiq initializer on appl
 ```ruby
 SidekiqUniqueJobs.configure do |config|
   config.logger = Sidekiq.logger # default, change at your own discretion
+  config.logger_enabled  = true # default, disable for test environments
   config.debug_lua       = false # Turn on when debugging
   config.lock_info       = false # Turn on when debugging
   config.lock_ttl        = 600   # Expire locks after 10 minutes

data/lib/sidekiq_unique_jobs/config.rb

@@ -8,6 +8,7 @@ module SidekiqUniqueJobs
                                                    :enabled,
                                                    :lock_prefix,
                                                    :logger,
+                                                   :logger_enabled,
                                                    :locks,
                                                    :strategies,
                                                    :debug_lua,
@@ -91,6 +92,9 @@ module SidekiqUniqueJobs
     # @return [nil]
     LOCK_TTL              = nil
     #
+    # @return [true,false] by default false (don't disable logger)
+    LOGGER_ENABLED        = true
+    #
     # @return [true] by default the gem is enabled
     ENABLED               = true
     #
@@ -180,6 +184,7 @@ module SidekiqUniqueJobs
         ENABLED,
         PREFIX,
         Sidekiq.logger,
+        LOGGER_ENABLED,
         LOCKS,
         STRATEGIES,
         DEBUG_LUA,

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -298,10 +298,15 @@ module SidekiqUniqueJobs
     # @api private
     #
     def brpoplpush(conn, wait)
+      # passing timeout 0 to brpoplpush causes it to block indefinitely
       raise InvalidArgument, "wait must be an integer" unless wait.is_a?(Integer)
+      return conn.brpoplpush(key.queued, key.primed, wait) if conn.class.to_s == "Redis::Namespace"
 
-      # passing timeout 0 to brpoplpush causes it to block indefinitely
-      conn.brpoplpush(key.queued, key.primed, timeout: wait)
+      if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && conn.respond_to?(:blmove)
+        conn.blmove(key.queued, key.primed, "RIGHT", "LEFT", timeout: wait)
+      else
+        conn.brpoplpush(key.queued, key.primed, timeout: wait)
+      end
     end
 
     #
@@ -370,5 +375,9 @@ module SidekiqUniqueJobs
         TIME => now_f,
       )
     end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
   end
 end

data/lib/sidekiq_unique_jobs/logging.rb

@@ -30,6 +30,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_debug(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.debug(message, &block)
       nil
@@ -45,6 +47,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_info(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.info(message, &block)
       nil
@@ -60,6 +64,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_warn(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.warn(message, &block)
       nil
@@ -75,6 +81,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_error(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.error(message, &block)
       nil
@@ -90,6 +98,8 @@ module SidekiqUniqueJobs
     # @yield [String, Exception] the message or exception to use for log message
     #
     def log_fatal(message_or_exception = nil, item = nil, &block)
+      return unless logging?
+
       message = build_message(message_or_exception, item)
       logger.fatal(message, &block)
 
@@ -218,5 +228,9 @@ module SidekiqUniqueJobs
 
       yield
     end
+
+    def logging?
+      SidekiqUniqueJobs.logging?
+    end
   end
 end

data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua

@@ -0,0 +1,92 @@
+-------- BEGIN keys ---------
+local digest           = KEYS[1]
+local queued           = KEYS[2]
+local primed           = KEYS[3]
+local locked           = KEYS[4]
+local info             = KEYS[5]
+local changelog        = KEYS[6]
+local digests          = KEYS[7]
+local expiring_digests = KEYS[8]
+-------- END keys ---------
+
+
+-------- BEGIN lock arguments ---------
+local job_id       = ARGV[1]
+local pttl         = tonumber(ARGV[2])
+local lock_type    = ARGV[3]
+local limit        = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+---------  BEGIN lock.lua ---------
+log_debug("BEGIN lock digest:", digest, "job_id:", job_id)
+
+if redis.call("HEXISTS", locked, job_id) == 1 then
+  log_debug(locked, "already locked with job_id:", job_id)
+  log("Duplicate")
+
+  log_debug("LREM", queued, -1, job_id)
+  redis.call("LREM", queued, -1, job_id)
+
+  log_debug("LREM", primed, 1, job_id)
+  redis.call("LREM", primed, 1, job_id)
+
+  return job_id
+end
+
+local locked_count   = redis.call("HLEN", locked)
+local within_limit   = limit > locked_count
+local limit_exceeded = not within_limit
+
+if limit_exceeded then
+  log_debug("Limit exceeded:", digest, "(",  locked_count, "of", limit, ")")
+  log("Limited")
+  return nil
+end
+
+log_debug("ZADD", expiring_digests, current_time + pttl, digest)
+redis.call("ZADD", expiring_digests, current_time + pttl, digest)
+
+log_debug("HSET", locked, job_id, current_time)
+redis.call("HSET", locked, job_id, current_time)
+
+log_debug("LREM", queued, -1, job_id)
+redis.call("LREM", queued, -1, job_id)
+
+log_debug("LREM", primed, 1, job_id)
+redis.call("LREM", primed, 1, job_id)
+
+-- The Sidekiq client sets pttl
+log_debug("PEXPIRE", digest, pttl)
+redis.call("PEXPIRE", digest, pttl)
+
+log_debug("PEXPIRE", locked, pttl)
+redis.call("PEXPIRE", locked, pttl)
+
+log_debug("PEXPIRE", info, pttl)
+redis.call("PEXPIRE", info, pttl)
+
+log_debug("PEXPIRE", queued, 1000)
+redis.call("PEXPIRE", queued, 1000)
+
+log_debug("PEXPIRE", primed, 1000)
+redis.call("PEXPIRE", primed, 1000)
+
+log("Locked")
+log_debug("END lock digest:", digest, "job_id:", job_id)
+return job_id
+----------  END lock.lua  ----------

data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb

@@ -71,6 +71,16 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
     config.logger = other
   end
 
+  #
+  # Check if logging is enabled
+  #
+  #
+  # @return [true, false]
+  #
+  def logging?
+    config.logger_enabled
+  end
+
   #
   # Temporarily use another configuration and reset to the old config after yielding
   #

data/lib/sidekiq_unique_jobs/version.rb

@@ -3,5 +3,5 @@
 module SidekiqUniqueJobs
   #
   # @return [String] the current SidekiqUniqueJobs version
-  VERSION = "7.1.26"
+  VERSION = "7.1.27"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 7.1.26
+  version: 7.1.27
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-28 00:00:00.000000000 Z
+date: 2022-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: brpoplpush-redis_script
@@ -145,6 +145,7 @@ files:
 - lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua
 - lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua
 - lib/sidekiq_unique_jobs/lua/lock.lua
+- lib/sidekiq_unique_jobs/lua/lock_until_expired.lua
 - lib/sidekiq_unique_jobs/lua/locked.lua
 - lib/sidekiq_unique_jobs/lua/queue.lua
 - lib/sidekiq_unique_jobs/lua/reap_orphans.lua