sidekiq-unique-jobs 7.1.25 → 7.1.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a5c0d9d30f11e61d3cc7599343951d242c5ccba2c93ee2a46138eab1037db81
-  data.tar.gz: a0721afd3b1a4e98655f9d1aba34e609d9ddd71bf6fd02d2b6f35bfbf4f9bece
+  metadata.gz: 501cb34e3378a86e0046733d3b948dd61ccf3faafc876433d2e1bf19125122b5
+  data.tar.gz: 628a44c9828484baf678b88437f76f056b4c31e549e68a853441280de98b41b4
 SHA512:
-  metadata.gz: 7751e266ae6df52925ba2b0b3fbc72e7fff5e2e979ae00dcc8fb7af433aac22627901a650cdd53f64479af440e68a814ad3d67f446509e4aef4351e6eed71da0
-  data.tar.gz: 210d306f47c106bab3ae6e109c24d728baf5a81cc6caa3ee309a7c96f547c5cd8503e99cb7f814671c1feccfdaa48ea992581c9f978bb2bfd662c51946e13ffd
+  metadata.gz: 2b3b98d114a3f328eb63a99409d6a938897f4d96cc6bdc1bc048c02452f09e50afe42b6779d1f6831bbe269abbc7e896d4d47c1c9b3710dea472c34d66c283d6
+  data.tar.gz: 7a12231a8b638821fd989e5b23998f29df4e401eda3308a4e3d5a496b3a8e982af6056dae9275aca97ef76a0cec1353dc8c200a916d3db7cc46b4fa9e9b3325a

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [v7.1.25](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.25) (2022-06-13)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.24...v7.1.25)
+
+**Fixed bugs:**
+
+- Fix: Include the correct middleware [\#716](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/716) ([mhenrixon](https://github.com/mhenrixon))
+
 ## [v7.1.24](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.24) (2022-06-09)
 
 [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.23...v7.1.24)

data/lib/sidekiq_unique_jobs/batch_delete.rb

@@ -91,6 +91,7 @@ module SidekiqUniqueJobs
           chunk.each do |digest|
             del_digest(pipeline, digest)
             pipeline.zrem(SidekiqUniqueJobs::DIGESTS, digest)
+            pipeline.zrem(SidekiqUniqueJobs::EXPIRING_DIGESTS, digest)
             @count += 1
           end
         end

data/lib/sidekiq_unique_jobs/cli.rb

@@ -20,9 +20,11 @@ module SidekiqUniqueJobs
     option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
     # :nodoc:
     def list(pattern = "*")
-      entries = digests.entries(pattern: pattern, count: options[:count])
-      say "Found #{entries.size} digests matching '#{pattern}':"
-      print_in_columns(entries.sort) if entries.any?
+      max_count = options[:count]
+      say "Searching for regular digests"
+      list_entries(digests.entries(pattern: pattern, count: max_count), pattern)
+      say "Searching for expiring digests"
+      list_entries(expiring_digests.entries(pattern: pattern, count: max_count), pattern)
     end
 
     desc "del PATTERN", "deletes unique digests from redis by pattern"
@@ -32,11 +34,9 @@ module SidekiqUniqueJobs
     def del(pattern)
       max_count = options[:count]
       if options[:dry_run]
-        result = digests.entries(pattern: pattern, count: max_count)
-        say "Would delete #{result.size} digests matching '#{pattern}'"
+        count_entries_for_del(max_count, pattern)
       else
-        deleted_count = digests.delete_by_pattern(pattern, count: max_count)
-        say "Deleted #{deleted_count} digests matching '#{pattern}'"
+        del_entries(max_count, pattern)
       end
     end
 
@@ -51,12 +51,17 @@ module SidekiqUniqueJobs
       console_class.start
     end
 
-    no_commands do
+    no_commands do # rubocop:disable Metrics/BlockLength
       # :nodoc:
       def digests
         @digests ||= SidekiqUniqueJobs::Digests.new
       end
 
+      # :nodoc:
+      def expiring_digests
+        @expiring_digests ||= SidekiqUniqueJobs::ExpiringDigests.new
+      end
+
       # :nodoc:
       def console_class
         require "pry"
@@ -65,6 +70,26 @@ module SidekiqUniqueJobs
         require "irb"
         IRB
       end
+
+      # :nodoc:
+      def list_entries(entries, pattern)
+        say "Found #{entries.size} digests matching '#{pattern}':"
+        print_in_columns(entries.sort) if entries.any?
+      end
+
+      # :nodoc:
+      def count_entries_for_del(max_count, pattern)
+        count = digests.entries(pattern: pattern, count: max_count).size +
+                expiring_digests.entries(pattern: pattern, count: max_count).size
+        say "Would delete #{count} digests matching '#{pattern}'"
+      end
+
+      # :nodoc:
+      def del_entries(max_count, pattern)
+        deleted_count = digests.delete_by_pattern(pattern, count: max_count).to_i +
+                        expiring_digests.delete_by_pattern(pattern, count: max_count).to_i
+        say "Deleted #{deleted_count} digests matching '#{pattern}'"
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/constants.rb

@@ -14,6 +14,7 @@ module SidekiqUniqueJobs
   CREATED_AT            = "created_at"
   DEAD_VERSION          = "uniquejobs:dead"
   DIGESTS               = "uniquejobs:digests"
+  EXPIRING_DIGESTS      = "uniquejobs:expiring_digests"
   ERRORS                = "errors"
   JID                   = "jid"
   LIMIT                 = "limit"

data/lib/sidekiq_unique_jobs/digests.rb

@@ -14,8 +14,8 @@ module SidekiqUniqueJobs
     # @return [String] the default pattern to use for matching
     SCAN_PATTERN  = "*"
 
-    def initialize
-      super(DIGESTS)
+    def initialize(digests_key = DIGESTS)
+      super(digests_key)
     end
 
     #

data/lib/sidekiq_unique_jobs/expiring_digests.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class ExpiringDigests provides access to the expiring digests used by until_expired locks
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class ExpiringDigests < Digests
+    def initialize
+      super(EXPIRING_DIGESTS)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/key.rb

@@ -33,6 +33,10 @@ module SidekiqUniqueJobs
     # @!attribute [r] digests
     #   @return [String] the zset with locked digests
     attr_reader :digests
+    #
+    # @!attribute [r] expiring_digests
+    #   @return [String] the zset with locked expiring_digests
+    attr_reader :expiring_digests
 
     #
     # Initialize a new Key
@@ -40,13 +44,14 @@ module SidekiqUniqueJobs
     # @param [String] digest the digest to use as key
     #
     def initialize(digest)
-      @digest    = digest
-      @queued    = suffixed_key("QUEUED")
-      @primed    = suffixed_key("PRIMED")
-      @locked    = suffixed_key("LOCKED")
-      @info      = suffixed_key("INFO")
-      @changelog = CHANGELOGS
-      @digests   = DIGESTS
+      @digest           = digest
+      @queued           = suffixed_key("QUEUED")
+      @primed           = suffixed_key("PRIMED")
+      @locked           = suffixed_key("LOCKED")
+      @info             = suffixed_key("INFO")
+      @changelog        = CHANGELOGS
+      @digests          = DIGESTS
+      @expiring_digests = EXPIRING_DIGESTS
     end
 
     #
@@ -81,7 +86,7 @@ module SidekiqUniqueJobs
     # @return [Array] an ordered array with all keys
     #
     def to_a
-      [digest, queued, primed, locked, info, changelog, digests]
+      [digest, queued, primed, locked, info, changelog, digests, expiring_digests]
     end
 
     private

data/lib/sidekiq_unique_jobs/lock.rb

@@ -66,7 +66,7 @@ module SidekiqUniqueJobs
           pipeline.set(key.digest, job_id)
           pipeline.hset(key.locked, job_id, now_f)
           info.set(lock_info, pipeline)
-          pipeline.zadd(key.digests, now_f, key.digest)
+          add_digest_to_set(pipeline, lock_info)
           pipeline.zadd(key.changelog, now_f, changelog_json(job_id, "queue.lua", "Queued"))
           pipeline.zadd(key.changelog, now_f, changelog_json(job_id, "lock.lua", "Locked"))
         end
@@ -321,5 +321,22 @@ module SidekiqUniqueJobs
         time: now_f,
       )
     end
+
+    #
+    # Add the digest to the correct sorted set
+    #
+    # @param [Object] pipeline a redis pipeline object for issue commands
+    # @param [Hash] lock_info the lock info relevant to the digest
+    #
+    # @return [nil]
+    #
+    def add_digest_to_set(pipeline, lock_info)
+      digest_string = key.digest
+      if lock_info["lock"] == :until_expired
+        pipeline.zadd(key.expiring_digests, now_f + lock_info["ttl"], digest_string)
+      else
+        pipeline.zadd(key.digests, now_f, digest_string)
+      end
+    end
   end
 end

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -32,6 +32,7 @@ module SidekiqUniqueJobs
     #
     # @return [Float] used to take into consideration the inaccuracy of redis timestamps
     CLOCK_DRIFT_FACTOR = 0.01
+    NETWORK_FACTOR = 0.04
 
     #
     # @!attribute [r] key
@@ -184,6 +185,7 @@ module SidekiqUniqueJobs
     #
     # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
     # @param [Method] primed_method reference to the method to use for getting a primed token
+    # @param [nil, Integer, Float] time to wait before timeout
     #
     # @yieldparam [string] job_id the sidekiq JID
     # @yieldreturn [void] whatever the calling block returns
@@ -239,9 +241,18 @@ module SidekiqUniqueJobs
     # @return [Object] whatever the block returns when lock was acquired
     #
     def primed_async(conn, wait = nil, &block)
+      timeout = (wait || config.timeout).to_i
+      timeout = 1 if timeout.zero?
+
+      brpoplpush_timeout = timeout
+      concurrent_timeout = add_drift(timeout)
+
+      reflect(:debug, :timeouts, item,
+              timeouts: { brpoplpush_timeout: brpoplpush_timeout, concurrent_timeout: concurrent_timeout })
+
       primed_jid = Concurrent::Promises
-                   .future(conn) { |red_con| pop_queued(red_con, wait) }
-                   .value(add_drift(wait || config.timeout))
+                   .future(conn) { |red_con| pop_queued(red_con, timeout) }
+                   .value
 
       handle_primed(primed_jid, &block)
     end
@@ -273,7 +284,7 @@ module SidekiqUniqueJobs
     #
     # @return [String] a previously enqueued token (now taken off the queue)
     #
-    def pop_queued(conn, wait = nil)
+    def pop_queued(conn, wait = 1)
       wait ||= config.timeout if config.wait_for_lock?
 
       if wait.nil?

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -1,11 +1,12 @@
 -------- BEGIN keys ---------
-local digest    = KEYS[1]
-local queued    = KEYS[2]
-local primed    = KEYS[3]
-local locked    = KEYS[4]
-local info      = KEYS[5]
-local changelog = KEYS[6]
-local digests   = KEYS[7]
+local digest           = KEYS[1]
+local queued           = KEYS[2]
+local primed           = KEYS[3]
+local locked           = KEYS[4]
+local info             = KEYS[5]
+local changelog        = KEYS[6]
+local digests          = KEYS[7]
+local expiring_digests = KEYS[8]
 -------- END keys ---------
 
 
@@ -57,8 +58,13 @@ if limit_exceeded then
   return nil
 end
 
-log_debug("ZADD", digests, current_time, digest)
-redis.call("ZADD", digests, current_time, digest)
+if lock_type == "until_expired" and pttl and pttl > 0 then
+  log_debug("ZADD", expiring_digests, current_time + pttl, digest)
+  redis.call("ZADD", expiring_digests, current_time + pttl, digest)
+else
+  log_debug("ZADD", digests, current_time, digest)
+  redis.call("ZADD", digests, current_time, digest)
+end
 
 log_debug("HSET", locked, job_id, current_time)
 redis.call("HSET", locked, job_id, current_time)

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -1,9 +1,10 @@
 redis.replicate_commands()
 
 -------- BEGIN keys ---------
-local digests_set  = KEYS[1]
-local schedule_set = KEYS[2]
-local retry_set    = KEYS[3]
+local digests_set          = KEYS[1]
+local expiring_digests_set = KEYS[2]
+local schedule_set         = KEYS[3]
+local retry_set            = KEYS[4]
 --------  END keys  ---------
 
 -------- BEGIN argv ---------
@@ -90,5 +91,32 @@ repeat
   index = index + per
 until index >= total or del_count >= reaper_count
 
+if del_count < reaper_count then
+  index = 0
+  total = redis.call("ZCOUNT", expiring_digests_set, 0, current_time)
+  repeat
+    local digests = redis.call("ZRANGEBYSCORE", expiring_digests_set, 0, current_time, "LIMIT", index, index + per -1)
+
+    for _, digest in pairs(digests) do
+      local queued     = digest .. ":QUEUED"
+      local primed     = digest .. ":PRIMED"
+      local locked     = digest .. ":LOCKED"
+      local info       = digest .. ":INFO"
+      local run_digest = digest .. ":RUN"
+      local run_queued = digest .. ":RUN:QUEUED"
+      local run_primed = digest .. ":RUN:PRIMED"
+      local run_locked = digest .. ":RUN:LOCKED"
+      local run_info   = digest .. ":RUN:INFO"
+
+      redis.call(del_cmd, digest, queued, primed, locked, info, run_digest, run_queued, run_primed, run_locked, run_info)
+
+      redis.call("ZREM", expiring_digests_set, digest)
+      del_count = del_count + 1
+    end
+
+    index = index + per
+  until index >= total or del_count >= reaper_count
+end
+
 log_debug("END")
 return del_count

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -85,6 +85,11 @@ if locked_count and locked_count < 1 then
   redis.call(del_cmd, locked)
 end
 
+if redis.call("LLEN", primed) == 0 then
+  log_debug(del_cmd, primed)
+  redis.call(del_cmd, primed)
+end
+
 if limit and limit <= 1 and locked_count and locked_count <= 1 then
   log_debug("ZREM", digests, digest)
   redis.call("ZREM", digests, digest)

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -20,7 +20,7 @@ module SidekiqUniqueJobs
         call_script(
           :reap_orphans,
           conn,
-          keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
+          keys: [DIGESTS, EXPIRING_DIGESTS, SCHEDULE, RETRY, PROCESSES],
           argv: [reaper_count, (Time.now - reaper_timeout).to_f],
         )
       end

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -34,9 +34,15 @@ module SidekiqUniqueJobs
 
       #
       # @!attribute [r] start_time
-      #   @return [Integer] The timestamp this execution started represented as integer
+      #   @return [Integer] The timestamp this execution started represented as Time (used for locks)
       attr_reader :start_time
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The clock stamp this execution started represented as integer
+      #      (used for redis compatibility as it is more accurate than time)
+      attr_reader :start_source
+
       #
       # @!attribute [r] timeout_ms
       #   @return [Integer] The allowed ms before timeout
@@ -49,11 +55,12 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests    = SidekiqUniqueJobs::Digests.new
-        @scheduled  = Redis::SortedSet.new(SCHEDULE)
-        @retried    = Redis::SortedSet.new(RETRY)
-        @start_time = time_source.call
-        @timeout_ms = SidekiqUniqueJobs.config.reaper_timeout * 1000
+        @digests      = SidekiqUniqueJobs::Digests.new
+        @scheduled    = Redis::SortedSet.new(SCHEDULE)
+        @retried      = Redis::SortedSet.new(RETRY)
+        @start_time   = Time.now
+        @start_source = time_source.call
+        @timeout_ms   = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -65,9 +72,20 @@ module SidekiqUniqueJobs
       def call
         return if queues_very_full?
 
+        BatchDelete.call(expired_digests, conn)
         BatchDelete.call(orphans, conn)
       end
 
+      def expired_digests
+        max_score = (start_time - reaper_timeout).to_f
+
+        if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && VersionCheck.satisfied?(::Redis::VERSION, ">= 4.6.0")
+          conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
+        else
+          conn.zrangebyscore(EXPIRING_DIGESTS, 0, max_score)
+        end
+      end
+
       #
       # Find orphaned digests
       #
@@ -104,7 +122,7 @@ module SidekiqUniqueJobs
       end
 
       def elapsed_ms
-        time_source.call - start_time
+        time_source.call - start_source
       end
 
       #
@@ -229,6 +247,7 @@ module SidekiqUniqueJobs
 
         loop do
           range_start = (page * page_size) - deleted_size
+
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1
@@ -238,6 +257,9 @@ module SidekiqUniqueJobs
           entries.each(&block)
 
           deleted_size = initial_size - conn.llen(queue_key)
+
+          # The queue is growing, not shrinking, just keep looping
+          deleted_size = 0 if deleted_size.negative?
         end
       end
 

data/lib/sidekiq_unique_jobs/testing.rb

@@ -90,15 +90,6 @@ module Sidekiq
         super(options)
       end
 
-      #
-      # Clears all jobs for this worker and removes all locks
-      #
-      def clear_all
-        super
-
-        SidekiqUniqueJobs::Digests.new.delete_by_pattern("*", count: 10_000)
-      end
-
       #
       # Prepends deletion of locks to clear
       #
@@ -124,5 +115,21 @@ module Sidekiq
     module ClassMethods
       prepend Overrides::ClassMethods
     end
+
+    #
+    # Prepends singleton methods to Sidekiq::Worker
+    #
+    module SignletonOverrides
+      #
+      # Clears all jobs for this worker and removes all locks
+      #
+      def clear_all
+        super
+
+        SidekiqUniqueJobs::Digests.new.delete_by_pattern("*", count: 10_000)
+      end
+    end
+
+    singleton_class.prepend SignletonOverrides
   end
 end

data/lib/sidekiq_unique_jobs/version.rb

@@ -3,5 +3,5 @@
 module SidekiqUniqueJobs
   #
   # @return [String] the current SidekiqUniqueJobs version
-  VERSION = "7.1.25"
+  VERSION = "7.1.26"
 end

data/lib/sidekiq_unique_jobs/web/helpers.rb

@@ -51,6 +51,16 @@ module SidekiqUniqueJobs
         @digests ||= SidekiqUniqueJobs::Digests.new
       end
 
+      #
+      # The collection of digests
+      #
+      #
+      # @return [SidekiqUniqueJobs::ExpiringDigests] the sorted set with expiring digests
+      #
+      def expiring_digests
+        @expiring_digests ||= SidekiqUniqueJobs::ExpiringDigests.new
+      end
+
       #
       # The collection of changelog entries
       #

data/lib/sidekiq_unique_jobs/web.rb

@@ -8,7 +8,7 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Web
-    def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       app.helpers do
         include Web::Helpers
       end
@@ -49,8 +49,25 @@ module SidekiqUniqueJobs
         erb(unique_template(:locks))
       end
 
+      app.get "/expiring_locks" do
+        @filter         = params[:filter] || "*"
+        @filter         = "*" if @filter == ""
+        @count          = (params[:count] || 100).to_i
+        @current_cursor = params[:cursor]
+        @prev_cursor    = params[:prev_cursor]
+
+        @total_size, @next_cursor, @locks = expiring_digests.page(
+          cursor: @current_cursor,
+          pattern: @filter,
+          page_size: @count,
+        )
+
+        erb(unique_template(:locks))
+      end
+
       app.get "/locks/delete_all" do
         digests.delete_by_pattern("*", count: digests.count)
+        expiring_digests.delete_by_pattern("*", count: digests.count)
         redirect_to :locks
       end
 
@@ -63,6 +80,7 @@ module SidekiqUniqueJobs
 
       app.get "/locks/:digest/delete" do
         digests.delete_by_digest(params[:digest])
+        expiring_digests.delete_by_digest(params[:digest])
         redirect_to :locks
       end
 
@@ -82,8 +100,9 @@ begin
   require "sidekiq/web" unless defined?(Sidekiq::Web)
 
   Sidekiq::Web.register(SidekiqUniqueJobs::Web)
-  Sidekiq::Web.tabs["Locks"]      = "locks"
-  Sidekiq::Web.tabs["Changelogs"] = "changelogs"
+  Sidekiq::Web.tabs["Locks"]          = "locks"
+  Sidekiq::Web.tabs["Expiring Locks"] = "expiring_locks"
+  Sidekiq::Web.tabs["Changelogs"]     = "changelogs"
   Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
 rescue NameError, LoadError => ex
   SidekiqUniqueJobs.logger.error(ex)

data/lib/sidekiq_unique_jobs.rb

@@ -72,6 +72,7 @@ require "sidekiq_unique_jobs/sidekiq_unique_ext"
 require "sidekiq_unique_jobs/on_conflict"
 require "sidekiq_unique_jobs/changelog"
 require "sidekiq_unique_jobs/digests"
+require "sidekiq_unique_jobs/expiring_digests"
 
 require "sidekiq_unique_jobs/config"
 require "sidekiq_unique_jobs/sidekiq_unique_jobs"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 7.1.25
+  version: 7.1.26
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-14 00:00:00.000000000 Z
+date: 2022-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: brpoplpush-redis_script
@@ -116,6 +116,7 @@ files:
 - lib/sidekiq_unique_jobs/deprecation.rb
 - lib/sidekiq_unique_jobs/digests.rb
 - lib/sidekiq_unique_jobs/exceptions.rb
+- lib/sidekiq_unique_jobs/expiring_digests.rb
 - lib/sidekiq_unique_jobs/job.rb
 - lib/sidekiq_unique_jobs/json.rb
 - lib/sidekiq_unique_jobs/key.rb