sidekiq-unique-jobs 6.0.24 → 7.0.4

data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb

@@ -3,7 +3,7 @@
 module SidekiqUniqueJobs
   # Module with convenience methods for the Sidekiq::Worker class
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module SidekiqWorkerMethods
     # Avoids duplicating worker_class.respond_to? in multiple places
     # @return [true, false]
@@ -16,7 +16,7 @@ module SidekiqUniqueJobs
     def worker_options
       return {} unless sidekiq_worker_class?
 
-      worker_class.get_sidekiq_options.stringify_keys
+      worker_class.get_sidekiq_options.deep_stringify_keys
     end
 
     # Tests that the
@@ -29,7 +29,7 @@ module SidekiqUniqueJobs
     # The Sidekiq::Worker implementation
     # @return [Sidekiq::Worker]
     def worker_class
-      @_worker_class ||= worker_class_constantize(@worker_class) # rubocop:disable Naming/MemoizedInstanceVariableName
+      @_worker_class ||= worker_class_constantize # rubocop:disable Naming/MemoizedInstanceVariableName
     end
 
     # The hook to call after a successful unlock
@@ -48,9 +48,9 @@ module SidekiqUniqueJobs
     # failing back to the original argument when the constant can't be found
     #
     # @return [Sidekiq::Worker]
-    def worker_class_constantize(klazz)
+    def worker_class_constantize(klazz = @worker_class)
       return klazz.class if klazz.is_a?(Sidekiq::Worker) # sidekiq v6.x
-      return klazz       unless klazz.is_a?(String)
+      return klazz unless klazz.is_a?(String)
 
       Object.const_get(klazz)
     rescue NameError => ex
@@ -62,6 +62,12 @@ module SidekiqUniqueJobs
       end
     end
 
+    #
+    # Returns the default worker options from Sidekiq
+    #
+    #
+    # @return [Hash<Symbol, Object>]
+    #
     def default_worker_options
       Sidekiq.default_worker_options
     end

data/lib/sidekiq_unique_jobs/testing.rb

@@ -1,36 +1,69 @@
 # frozen_string_literal: true
 
 # :nocov:
+# :nodoc:
 
 require "sidekiq"
 require "sidekiq/testing"
+require "sidekiq_unique_jobs/rspec/matchers"
+require "sidekiq_unique_jobs/lock/validator"
+require "sidekiq_unique_jobs/lock/client_validator"
+require "sidekiq_unique_jobs/lock/server_validator"
 
+#
+# See Sidekiq gem for more details
+#
 module Sidekiq
+  #
+  # Temporarily turn Sidekiq's options into something different
+  #
+  # @note this method will restore the original options after yielding
+  #
+  # @param [Hash<Symbol, Object>] tmp_config the temporary config to use
+  #
   def self.use_options(tmp_config = {})
-    old_config = default_worker_options
+    old_options = default_worker_options.dup
+
     default_worker_options.clear
     self.default_worker_options = tmp_config
-
     yield
   ensure
     default_worker_options.clear
-    self.default_worker_options = old_config
+    self.default_worker_options = DEFAULT_WORKER_OPTIONS
+    self.default_worker_options = old_options
   end
 
+  #
+  # See Sidekiq::Worker in Sidekiq gem for more details
+  #
   module Worker
+    #
+    # Adds class methods to Sidekiq::Worker
+    #
     module ClassMethods
+      #
+      # Temporarily turn a workers sidekiq_options into something different
+      #
+      # @note this method will restore the original configuration after yielding
+      #
+      # @param [Hash<Symbol, Object>] tmp_config the temporary config to use
+      #
       def use_options(tmp_config = {})
-        old_config = get_sidekiq_options
-        sidekiq_options(tmp_config)
+        old_options = sidekiq_options_hash.dup
+        sidekiq_options(old_options.merge(tmp_config))
 
         yield
       ensure
-        sidekiq_options(old_config)
+        self.sidekiq_options_hash = Sidekiq::DEFAULT_WORKER_OPTIONS
+        sidekiq_options(old_options)
       end
 
+      #
+      # Clears the jobs for this worker and removes all locks
+      #
       def clear
         jobs.each do |job|
-          SidekiqUniqueJobs::Unlockable.delete(job)
+          SidekiqUniqueJobs::Unlockable.unlock(job)
         end
 
         Sidekiq::Queues[queue].clear
@@ -38,25 +71,33 @@ module Sidekiq
       end
     end
 
+    #
+    # Prepends deletion of locks to clear_all
+    #
     module Overrides
-      def self.included(base)
-        base.extend Testing
-        base.class_eval do
-          class << self
-            alias_method :clear_all_orig, :clear_all
-            alias_method :clear_all, :clear_all_ext
-          end
-        end
+      #
+      # Overrides sidekiq_options on the worker class to prepend validation
+      #
+      # @param [Hash] options worker options
+      #
+      # @return [void]
+      #
+      def sidekiq_options(options = {})
+        SidekiqUniqueJobs.validate_worker!(options) if SidekiqUniqueJobs.config.raise_on_config_error
+
+        super(options)
       end
 
-      module Testing
-        def clear_all_ext
-          clear_all_orig
-          SidekiqUniqueJobs::Util.del("*", 1000)
-        end
+      #
+      # Clears all jobs for this worker and removes all locks
+      #
+      def clear_all
+        super
+
+        SidekiqUniqueJobs::Digests.new.delete_by_pattern("*", count: 10_000)
       end
     end
 
-    include Overrides
+    prepend Overrides
   end
 end

data/lib/sidekiq_unique_jobs/timer_task.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # @see [Concurrent::TimerTask] https://www.rubydoc.info/gems/concurrent-ruby/Concurrent/TimerTask
+  #
+  class TimerTask < ::Concurrent::TimerTask
+    private
+
+    def ns_initialize(opts, &task)
+      set_deref_options(opts)
+
+      self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
+      self.timeout_interval = opts[:timeout] || opts[:timeout_interval] || TIMEOUT_INTERVAL
+      @run_now  = opts[:now] || opts[:run_now]
+      @executor = Concurrent::RubySingleThreadExecutor.new
+      @running  = Concurrent::AtomicBoolean.new(false)
+      @task     = task
+      @value    = nil
+
+      self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
+    end
+
+    def schedule_next_task(interval = execution_interval)
+      exec_task = ->(completion) { execute_task(completion) }
+      Concurrent::ScheduledTask.execute(interval, args: [Concurrent::Event.new], &exec_task)
+      nil
+    end
+
+    # @!visibility private
+    def execute_task(completion) # rubocop:disable Metrics/MethodLength
+      return nil unless @running.true?
+
+      timeout_task = -> { timeout_task(completion) }
+
+      Concurrent::ScheduledTask.execute(
+        timeout_interval,
+        args: [completion],
+        &timeout_task
+      )
+      @thread_completed = Concurrent::Event.new
+
+      @value = @reason  = nil
+      @executor.post do
+        @value = @task.call(self)
+      rescue Exception => ex # rubocop:disable Lint/RescueException
+        @reason = ex
+      ensure
+        @thread_completed.set
+      end
+
+      @thread_completed.wait
+
+      if completion.try?
+        schedule_next_task
+        time = Time.now
+        observers.notify_observers do
+          [time, value, @reason]
+        end
+      end
+      nil
+    end
+
+    # @!visibility private
+    def timeout_task(completion)
+      return unless @running.true?
+      return unless completion.try?
+
+      @executor.kill
+      @executor.wait_for_termination
+      @executor = Concurrent::RubySingleThreadExecutor.new
+
+      @thread_completed.set
+
+      schedule_next_task
+      observers.notify_observers(Time.now, nil, Concurrent::TimeoutError.new)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/timing.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Handles timing of things
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module Timing
+    module_function
+
+    #
+    # Used for timing method calls
+    #
+    #
+    # @return [yield return, Float]
+    #
+    def timed
+      start_time = time_source.call
+
+      [yield, time_source.call - start_time]
+    end
+
+    #
+    # Used to get a current representation of time as Integer
+    #
+    #
+    # @return [Integer]
+    #
+    def time_source
+      -> { (clock_stamp * 1000).to_i }
+    end
+
+    #
+    # Returns the current time as float
+    #
+    # @see SidekiqUniqueJobs.now_f
+    #
+    # @return [Float]
+    #
+    def now_f
+      SidekiqUniqueJobs.now_f
+    end
+
+    #
+    # Returns a float representation of the current time.
+    #   Either from Process or Time
+    #
+    #
+    # @return [Float]
+    #
+    def clock_stamp
+      if Process.const_defined?("CLOCK_MONOTONIC")
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      else
+        Time.now.to_f
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/unlockable.rb

@@ -4,23 +4,39 @@ module SidekiqUniqueJobs
   # Utility module to help manage unique keys in redis.
   # Useful for deleting keys that for whatever reason wasn't deleted
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Unlockable
     module_function
 
     # Unlocks a job.
     # @param [Hash] item a Sidekiq job hash
     def unlock(item)
-      SidekiqUniqueJobs::UniqueArgs.digest(item)
+      SidekiqUniqueJobs::Job.add_digest(item)
       SidekiqUniqueJobs::Locksmith.new(item).unlock
     end
 
-    # Deletes a lock regardless of if it was locked or not.
+    # Unlocks a job.
+    # @param [Hash] item a Sidekiq job hash
+    def unlock!(item)
+      SidekiqUniqueJobs::Job.add_digest(item)
+      SidekiqUniqueJobs::Locksmith.new(item).unlock!
+    end
+
+    # Deletes a lock unless it has ttl
     #
     # This is good for situations when a job is locked by another item
     # @param [Hash] item a Sidekiq job hash
     def delete(item)
-      SidekiqUniqueJobs::UniqueArgs.digest(item)
+      SidekiqUniqueJobs::Job.add_digest(item)
+      SidekiqUniqueJobs::Locksmith.new(item).delete
+    end
+
+    # Deletes a lock regardless of if it was locked or has ttl.
+    #
+    # This is good for situations when a job is locked by another item
+    # @param [Hash] item a Sidekiq job hash
+    def delete!(item)
+      SidekiqUniqueJobs::Job.add_digest(item)
       SidekiqUniqueJobs::Locksmith.new(item).delete!
     end
   end

data/lib/sidekiq_unique_jobs/update_version.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class UpdateVersion sets the right version in redis
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class UpdateVersion
+    #
+    # Sets the right versions in redis
+    #
+    # @note the version isn't used yet but will be for automatic upgrades
+    #
+    # @return [true] when version changed
+    #
+    def self.call
+      Script::Caller.call_script(
+        :update_version,
+        keys: [LIVE_VERSION, DEAD_VERSION],
+        argv: [SidekiqUniqueJobs.version],
+      )
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/upgrade_locks.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Upgrades locks between gem version upgrades
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class UpgradeLocks # rubocop:disable Metrics/ClassLength
+    #
+    # @return [Integer] the number of keys to batch upgrade
+    BATCH_SIZE = 100
+    #
+    # @return [Array<String>] suffixes for old version
+    OLD_SUFFIXES = %w[
+      GRABBED
+      AVAILABLE
+      EXISTS
+      VERSION
+    ].freeze
+
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Connection
+
+    #
+    # Performs upgrade of old locks
+    #
+    #
+    # @return [Integer] the number of upgrades locks
+    #
+    def self.call
+      redis do |conn|
+        new(conn).call
+      end
+    end
+
+    attr_reader :conn
+
+    def initialize(conn)
+      @count         = 0
+      @conn          = conn
+      redis_version # Avoid pipelined calling redis_version and getting a future.
+    end
+
+    #
+    # Performs upgrade of old locks
+    #
+    #
+    # @return [Integer] the number of upgrades locks
+    #
+    def call
+      with_logging_context do
+        return log_info("Already upgraded to #{version}") if conn.hget(upgraded_key, version)
+        # TODO: Needs handling of v7.0.0 => v7.0.1 where we don't want to
+        return log_info("Skipping upgrade because #{DEAD_VERSION} has been set") if conn.get(DEAD_VERSION)
+
+        log_info("Start - Upgrading Locks")
+
+        upgrade_v6_locks
+        delete_unused_v6_keys
+        delete_supporting_v6_keys
+
+        conn.hset(upgraded_key, version, now_f)
+        log_info("Done - Upgrading Locks")
+      end
+
+      @count
+    end
+
+    private
+
+    def upgraded_key
+      @upgraded_key ||= "#{LIVE_VERSION}:UPGRADED"
+    end
+
+    def upgrade_v6_locks
+      log_info("Start - Converting v6 locks to v7")
+      conn.scan_each(match: "*:GRABBED", count: BATCH_SIZE) do |grabbed_key|
+        upgrade_v6_lock(grabbed_key)
+        @count += 1
+      end
+      log_info("Done - Converting v6 locks to v7")
+    end
+
+    def upgrade_v6_lock(grabbed_key)
+      locked_key = grabbed_key.gsub(":GRABBED", ":LOCKED")
+      digest     = grabbed_key.gsub(":GRABBED", "")
+      locks      = conn.hgetall(grabbed_key)
+
+      conn.pipelined do
+        conn.hmset(locked_key, *locks.to_a)
+        conn.zadd(DIGESTS, locks.values.first, digest)
+      end
+    end
+
+    def delete_unused_v6_keys
+      log_info("Start - Deleting v6 keys")
+      OLD_SUFFIXES.each do |suffix|
+        delete_suffix(suffix)
+      end
+      log_info("Done - Deleting v6 keys")
+    end
+
+    def delete_supporting_v6_keys
+      batch_delete("unique:keys")
+    end
+
+    def delete_suffix(suffix)
+      batch_scan(match: "*:#{suffix}", count: BATCH_SIZE) do |keys|
+        batch_delete(*keys)
+      end
+    end
+
+    def batch_delete(*keys)
+      return if keys.empty?
+
+      conn.pipelined do
+        if VersionCheck.satisfied?(redis_version, ">= 4.0.0")
+          conn.unlink(*keys)
+        else
+          conn.del(*keys)
+        end
+      end
+    end
+
+    def batch_scan(match:, count:)
+      cursor = "0"
+      loop do
+        cursor, values = conn.scan(cursor, match: match, count: count)
+        yield values
+        break if cursor == "0"
+      end
+    end
+
+    def version
+      SidekiqUniqueJobs.version
+    end
+
+    def now_f
+      SidekiqUniqueJobs.now_f
+    end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
+
+    def logging_context
+      if logger_context_hash?
+        { "uniquejobs" => :upgrade_locks }
+      else
+        "uniquejobs-upgrade_locks"
+      end
+    end
+  end
+end