sidekiq-unique-jobs 5.0.11 → 6.0.0.rc1

data/lib/sidekiq_unique_jobs/timeout.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'timeout'
+
+module SidekiqUniqueJobs
+  module Timeout
+    def using_timeout(value)
+      ::Timeout.timeout(value) do
+        yield
+      end
+    end
+  end
+end
+
+require 'sidekiq_unique_jobs/timeout/calculator'

data/lib/sidekiq_unique_jobs/timeout/calculator.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Timeout
+    class Calculator
+      include SidekiqUniqueJobs::SidekiqWorkerMethods
+      attr_reader :item
+
+      def initialize(item)
+        @item         = item
+        @worker_class = item[CLASS_KEY]
+      end
+
+      def time_until_scheduled
+        return 0 unless scheduled_at
+        scheduled_at.to_i - Time.now.utc.to_i
+      end
+
+      def scheduled_at
+        @scheduled_at ||= item[AT_KEY]
+      end
+
+      def seconds
+        raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
+      end
+
+      def lock_expiration
+        @lock_expiration ||= begin
+          expiration = item[LOCK_EXPIRATION_KEY]
+          expiration ||= worker_options[LOCK_EXPIRATION_KEY]
+          expiration && expiration + time_until_scheduled
+        end
+      end
+
+      def lock_timeout
+        @lock_timeout = begin
+          timeout = default_worker_options[LOCK_TIMEOUT_KEY]
+          timeout = default_lock_timeout if default_lock_timeout
+          timeout = worker_options[LOCK_TIMEOUT_KEY] if worker_options.key?(LOCK_TIMEOUT_KEY)
+          timeout
+        end
+      end
+
+      def default_lock_timeout
+        SidekiqUniqueJobs.config.default_lock_timeout
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/unique_args.rb

@@ -5,101 +5,73 @@ require 'sidekiq_unique_jobs/normalizer'
 
 module SidekiqUniqueJobs
   # This class exists to be testable and the entire api should be considered private
-  # rubocop:disable ClassLength
   class UniqueArgs
     CLASS_NAME = 'SidekiqUniqueJobs::UniqueArgs'
-    extend Forwardable
 
-    def_delegators :SidekiqUniqueJobs, :config, :worker_class_constantize
-    def_delegators :Sidekiq, :logger
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::SidekiqWorkerMethods
 
     def self.digest(item)
       new(item).unique_digest
     end
 
+    attr_reader :item
+
     def initialize(item)
-      Sidekiq::Logging.with_context(CLASS_NAME) do
-        @item = item
-        @worker_class            ||= worker_class_constantize(@item[CLASS_KEY])
-        @item[UNIQUE_PREFIX_KEY] ||= unique_prefix
-        @item[UNIQUE_ARGS_KEY]     = unique_args(@item[ARGS_KEY])
-        @item[UNIQUE_DIGEST_KEY]   = unique_digest
-      end
+      @item         = item
+      @worker_class = item[CLASS_KEY]
+
+      add_uniqueness_to_item
+    end
+
+    def add_uniqueness_to_item
+      item[UNIQUE_PREFIX_KEY] ||= unique_prefix
+      item[UNIQUE_ARGS_KEY]     = unique_args(item[ARGS_KEY])
+      item[UNIQUE_DIGEST_KEY]   = unique_digest
     end
 
     def unique_digest
-      @unique_digest ||= begin
-        digest = Digest::MD5.hexdigest(Sidekiq.dump_json(digestable_hash))
-        digest = "#{unique_prefix}:#{digest}"
-        logger.debug { "#{__method__} : #{digestable_hash} into #{digest}" }
-        digest
-      end
+      @unique_digest ||= create_digest
+    end
+
+    def create_digest
+      digest = Digest::MD5.hexdigest(Sidekiq.dump_json(digestable_hash))
+      "#{unique_prefix}:#{digest}"
     end
 
     def unique_prefix
-      return config.unique_prefix unless sidekiq_worker_class?
-      @worker_class.get_sidekiq_options[UNIQUE_PREFIX_KEY] || config.unique_prefix
+      worker_options[UNIQUE_PREFIX_KEY] || SidekiqUniqueJobs.config.unique_prefix
     end
 
     def digestable_hash
       @item.slice(CLASS_KEY, QUEUE_KEY, UNIQUE_ARGS_KEY).tap do |hash|
-        if unique_on_all_queues?
-          logger.debug { "#{__method__} deleting queue: #{@item[QUEUE_KEY]}" }
-          hash.delete(QUEUE_KEY)
-        end
-        if unique_across_workers?
-          logger.debug { "#{__method__} deleting class: #{@item[CLASS_KEY]}" }
-          hash.delete(CLASS_KEY)
-        end
+        hash.delete(QUEUE_KEY) if unique_on_all_queues?
+        hash.delete(CLASS_KEY) if unique_across_workers?
       end
     end
 
     def unique_args(args)
-      if unique_args_enabled?
-        filtered_args(args)
-      else
-        logger.debug { "#{__method__} : unique arguments disabled" }
-        args
-      end
-    rescue NameError => ex
-      logger.error "#{__method__}(#{args}) : failed with (#{ex.message})"
-      logger.error ex
-
-      raise if config.raise_unique_args_errors
-
+      return filtered_args(args) if unique_args_enabled?
       args
     end
 
     def unique_on_all_queues?
-      return unless sidekiq_worker_class?
-      @item[UNIQUE_ON_ALL_QUEUES_KEY] || @worker_class.get_sidekiq_options[UNIQUE_ON_ALL_QUEUES_KEY]
+      item[UNIQUE_ON_ALL_QUEUES_KEY] || worker_options[UNIQUE_ON_ALL_QUEUES_KEY]
     end
 
     def unique_across_workers?
-      return unless sidekiq_worker_class?
-      @item[UNIQUE_ACROSS_WORKERS_KEY] || @worker_class.get_sidekiq_options[UNIQUE_ACROSS_WORKERS_KEY]
+      item[UNIQUE_ACROSS_WORKERS_KEY] || worker_options[UNIQUE_ACROSS_WORKERS_KEY]
     end
 
     def unique_args_enabled?
-      return unless sidekiq_worker_class?
       unique_args_method # && !unique_args_method.is_a?(Boolean)
     end
 
-    def sidekiq_worker_class?
-      if @worker_class.respond_to?(:get_sidekiq_options)
-        true
-      else
-        logger.debug { "#{__method__} #{@worker_class} does not respond to :get_sidekiq_options" }
-        nil
-      end
-    end
-
     # Filters unique arguments by proc or symbol
     # returns provided arguments for other configurations
     def filtered_args(args)
       return args if args.empty?
       json_args = Normalizer.jsonify(args)
-      logger.debug { "#filtered_args #{args} => #{json_args}" }
 
       case unique_args_method
       when Proc
@@ -107,44 +79,28 @@ module SidekiqUniqueJobs
       when Symbol
         filter_by_symbol(json_args)
       else
-        logger.debug { "#{__method__} arguments not filtered (using all arguments for uniqueness)" }
+        log_debug("#{__method__} arguments not filtered (using all arguments for uniqueness)")
         json_args
       end
     end
 
     def filter_by_proc(args)
-      if unique_args_method.nil?
-        logger.warn { "#{__method__} : unique_args_method is nil. Returning (#{args})" }
-        return args
-      end
-
-      filter_args = unique_args_method.call(args)
-      logger.debug { "#{__method__} : #{args} -> #{filter_args}" }
-      filter_args
+      unique_args_method.call(args)
     end
 
     def filter_by_symbol(args)
-      unless @worker_class.respond_to?(unique_args_method)
-        logger.warn do
-          "#{__method__} : #{@worker_class} does not respond to #{unique_args_method}). Returning (#{args})"
-        end
-        return args
-      end
+      return args unless worker_method_defined?(unique_args_method)
 
-      filter_args = @worker_class.send(unique_args_method, args)
-      logger.debug { "#{__method__} : #{unique_args_method}(#{args}) => #{filter_args}" }
-      filter_args
+      worker_class.send(unique_args_method, args)
     rescue ArgumentError => ex
-      logger.fatal "#{__method__} : #{@worker_class}'s #{unique_args_method} needs at least one argument"
-      logger.fatal ex
+      log_fatal(ex)
       args
     end
 
     def unique_args_method
-      @unique_args_method ||= @worker_class.get_sidekiq_options[UNIQUE_ARGS_KEY] if sidekiq_worker_class?
-      @unique_args_method ||= :unique_args if @worker_class.respond_to?(:unique_args)
+      @unique_args_method ||= worker_options[UNIQUE_ARGS_KEY]
+      @unique_args_method ||= :unique_args if worker_method_defined?(:unique_args)
       @unique_args_method ||= Sidekiq.default_worker_options.stringify_keys[UNIQUE_ARGS_KEY]
     end
   end
-  # rubocop:enable ClassLength
 end

data/lib/sidekiq_unique_jobs/unlockable.rb

@@ -5,22 +5,13 @@ module SidekiqUniqueJobs
     module_function
 
     def unlock(item)
-      return unless item[UNIQUE_DIGEST_KEY]
-      unlock_by_key(item[UNIQUE_DIGEST_KEY], item[JID_KEY])
+      SidekiqUniqueJobs::UniqueArgs.digest(item)
+      SidekiqUniqueJobs::Locksmith.new(item).unlock
     end
 
-    def unlock_by_key(unique_key, jid, redis_pool = nil)
-      lock_released = Scripts::ReleaseLock.execute(redis_pool, unique_key, jid)
-      ensure_job_id_removed(jid)
-      lock_released
-    end
-
-    def ensure_job_id_removed(jid)
-      Sidekiq.redis { |conn| conn.hdel(SidekiqUniqueJobs::HASH_KEY, jid) }
-    end
-
-    def logger
-      SidekiqUniqueJobs.logger
+    def delete(item)
+      SidekiqUniqueJobs::UniqueArgs.digest(item)
+      SidekiqUniqueJobs::Locksmith.new(item).delete!
     end
   end
 end

data/lib/sidekiq_unique_jobs/util.rb

@@ -1,85 +1,46 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  module Util # rubocop:disable Metrics/ModuleLength
+  module Util
     COUNT             = 'COUNT'
     DEFAULT_COUNT     = 1_000
     EXPIRE_BATCH_SIZE = 100
-    MATCH             = 'MATCH'
-    KEYS_METHOD       = 'keys'
-    SCAN_METHOD       = 'scan'
+    SCAN_METHOD       = 'SCAN'
     SCAN_PATTERN      = '*'
 
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Connection
     extend self # rubocop:disable Style/ModuleFunction
 
     def keys(pattern = SCAN_PATTERN, count = DEFAULT_COUNT)
-      send("keys_by_#{redis_keys_method}", pattern, count)
+      return redis(&:keys) if pattern.nil?
+      redis { |conn| conn.scan_each(match: prefix(pattern), count: count).to_a }
     end
 
-    def unique_key(jid)
-      connection do |conn|
-        conn.hget(SidekiqUniqueJobs::HASH_KEY, jid)
-      end
-    end
-
-    def del(pattern = SCAN_PATTERN, count = 0, dry_run = true)
+    # Deletes unique keys from redis
+    #
+    #
+    # @param pattern [String] a pattern to scan for in redis
+    # @param count [Integer] the maximum number of keys to delete
+    # @return [Boolean] report success
+    def del(pattern = SCAN_PATTERN, count = 0)
       raise ArgumentError, 'Please provide a number of keys to delete greater than zero' if count.zero?
-      logger.debug { "Deleting keys by: #{pattern}" }
-      keys, time = timed { keys(pattern, count) }
-      logger.debug { "#{keys.size} matching keys found in #{time} sec." }
-      keys = dry_run(keys)
-      logger.debug { "#{keys.size} matching keys after post-processing" }
-      unless dry_run
-        logger.debug { "deleting #{keys}..." }
-        _, time = timed { batch_delete(keys) }
-        logger.debug { "Deleted in #{time} sec." }
-      end
-      keys.size
-    end
-
-    def unique_hash
-      connection do |conn|
-        conn.hgetall(SidekiqUniqueJobs::HASH_KEY)
-      end
-    end
-
-    def expire # rubocop:disable Metrics/MethodLength
-      removed_keys = {}
-      connection do |conn|
-        cursor = '0'
-        loop do
-          cursor, jobs = get_jobs(conn, cursor)
-          jobs.each do |job_array|
-            jid, unique_key = job_array
-
-            next if conn.get(unique_key)
-            conn.hdel(SidekiqUniqueJobs::HASH_KEY, jid)
-            removed_keys[jid] = unique_key
-          end
+      pattern = "#{pattern}:*" unless pattern.end_with?(':*')
 
-          break if cursor == '0'
-        end
-      end
+      log_debug { "Deleting keys by: #{pattern}" }
+      keys, time = timed { keys(pattern, count) }
+      key_size   = keys.size
+      log_debug { "#{key_size} keys found in #{time} sec." }
+      _, time = timed { batch_delete(keys) }
+      log_debug { "Deleted #{key_size} keys in #{time} sec." }
 
-      removed_keys
+      key_size
     end
 
     private
 
-    def get_jobs(conn, cursor)
-      conn.hscan(SidekiqUniqueJobs::HASH_KEY, [cursor, MATCH, SCAN_PATTERN, COUNT, EXPIRE_BATCH_SIZE])
-    end
-
-    def keys_by_scan(pattern, count)
-      connection { |conn| conn.scan_each(match: prefix(pattern), count: count).to_a }
-    end
-
-    def keys_by_keys(pattern, _count)
-      connection { |conn| conn.keys(prefix(pattern)).to_a }
-    end
-
     def batch_delete(keys)
-      connection do |conn|
+      redis do |conn|
         keys.each_slice(500) do |chunk|
           conn.pipelined do
             chunk.each do |key|
@@ -90,22 +51,15 @@ module SidekiqUniqueJobs
       end
     end
 
-    def dry_run(keys, pattern = nil)
-      return keys if pattern.nil?
-      regex = Regexp.new(pattern)
-      keys.select { |k| regex.match k }
-    end
-
-    def timed(&_block)
-      start = Time.now
-      result = yield
-      elapsed = (Time.now - start).round(2)
+    def timed
+      start   = current_time
+      result  = yield
+      elapsed = (current_time - start).round(2)
       [result, elapsed]
     end
 
-    def prefix_keys(keys)
-      keys = Array(keys).flatten.compact
-      keys.map { |key| prefix(key) }
+    def current_time
+      Time.now
     end
 
     def prefix(key)
@@ -117,21 +71,5 @@ module SidekiqUniqueJobs
     def unique_prefix
       SidekiqUniqueJobs.config.unique_prefix
     end
-
-    def connection(&block)
-      SidekiqUniqueJobs.connection(&block)
-    end
-
-    def redis_version
-      SidekiqUniqueJobs.redis_version
-    end
-
-    def redis_keys_method
-      (redis_version >= '2.8') ? SCAN_METHOD : KEYS_METHOD
-    end
-
-    def logger
-      SidekiqUniqueJobs.logger
-    end
   end
 end

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  VERSION = '5.0.11'
+  VERSION = '6.0.0.rc1'
 end

data/redis/acquire_lock.lua

@@ -1,6 +1,6 @@
 local unique_key = KEYS[1]
 local job_id     = ARGV[1]
-local expires    = ARGV[2]
+local expires    = tonumber(ARGV[2])
 local stored_jid = redis.pcall('get', unique_key)
 
 if stored_jid then
@@ -11,8 +11,10 @@ if stored_jid then
   end
 end
 
-if redis.pcall('set', unique_key, job_id, 'nx', 'ex', expires) then
-  -- redis.pcall('hsetnx', 'uniquejobs', job_id, unique_key)
+if redis.call('SET', unique_key, job_id, 'nx') then
+  if expires then
+    redis.call('EXPIRE', unique_key, expires)
+  end
   return 1
 else
   return 0