sidekiq-unique-jobs 5.0.2 → 5.0.11

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
   module Lock
     class UntilExecuted
-      OK ||= 'OK'.freeze
+      OK ||= 'OK'
 
       include SidekiqUniqueJobs::Unlockable
 
@@ -36,9 +38,7 @@ module SidekiqUniqueJobs
       end
 
       def lock(scope)
-        if scope.to_sym != :client
-          raise ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
-        end
+        raise ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}" if scope.to_sym != :client
 
         Scripts::AcquireLock.execute(
           redis_pool,
@@ -47,7 +47,6 @@ module SidekiqUniqueJobs
           max_lock_time,
         )
       end
-      # rubocop:enable MethodLength
 
       def unique_key
         @unique_key ||= UniqueArgs.digest(item)

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
   module Lock
     class UntilExecuting < UntilExecuted

data/lib/sidekiq_unique_jobs/lock/until_timeout.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
   module Lock
     class UntilTimeout < UntilExecuted

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
   module Lock
     class WhileExecuting
@@ -7,22 +9,21 @@ module SidekiqUniqueJobs
 
       def initialize(item, redis_pool = nil)
         @item = item
-        @mutex = Mutex.new
         @redis_pool = redis_pool
         @unique_digest = "#{create_digest}:run"
+        @mutex = Mutex.new
       end
 
       def synchronize
-        @mutex.lock
-        sleep 0.1 until locked?
-
-        yield
+        @mutex.synchronize do
+          sleep 0.1 until locked?
+          yield
+        end
       rescue Sidekiq::Shutdown
         logger.fatal { "the unique_key: #{@unique_digest} needs to be unlocked manually" }
         raise
       ensure
-        SidekiqUniqueJobs.connection(@redis_pool) { |c| c.del @unique_digest }
-        @mutex.unlock
+        SidekiqUniqueJobs.connection(@redis_pool) { |conn| conn.del @unique_digest }
       end
 
       def locked?
@@ -42,8 +43,8 @@ module SidekiqUniqueJobs
       end
 
       def create_digest
-        @unique_digest ||= @item[UNIQUE_DIGEST_KEY]
-        @unique_digest ||= SidekiqUniqueJobs::UniqueArgs.digest(@item)
+        @create_digest ||= @item[UNIQUE_DIGEST_KEY]
+        @create_digest ||= SidekiqUniqueJobs::UniqueArgs.digest(@item)
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sidekiq_unique_jobs/lock/until_executed'
 require 'sidekiq_unique_jobs/lock/until_executing'
 require 'sidekiq_unique_jobs/lock/while_executing'

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sidekiq'
 
 module SidekiqUniqueJobs

data/lib/sidekiq_unique_jobs/normalizer.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 require 'json'
 
 module SidekiqUniqueJobs
   module Normalizer
     def self.jsonify(args)
-      JSON.parse(args.to_json)
+      Sidekiq.load_json(Sidekiq.dump_json(args))
     end
   end
 end

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
   module OptionsWithFallback
     def self.included(base)
-      base.class_attribute :lock_cache unless base.respond_to?(:lock_cache)
-      base.lock_cache ||= {}
+      base.send(:extend, SidekiqUniqueJobs::OptionsWithFallback::ClassMethods)
     end
 
     def unique_enabled?
@@ -22,7 +23,7 @@ module SidekiqUniqueJobs
     end
 
     def lock_class
-      lock_cache[unique_lock.to_sym] ||= "SidekiqUniqueJobs::Lock::#{unique_lock.to_s.classify}".constantize
+      lock_cache[unique_lock.to_sym] ||= Object.const_get("SidekiqUniqueJobs::Lock::#{unique_lock.to_s.classify}")
     end
 
     def unique_lock
@@ -45,11 +46,29 @@ module SidekiqUniqueJobs
       hash[key]
     end
 
+    def lock_cache
+      self.class.lock_cache
+    end
+
+    def lock_cache=(obj)
+      self.class.lock_cache = obj
+    end
+
     def options
       @options ||= worker_class.get_sidekiq_options if worker_class.respond_to?(:get_sidekiq_options)
       @options ||= Sidekiq.default_worker_options
       @options ||= {}
       @options &&= @options.stringify_keys
     end
+
+    module ClassMethods
+      def lock_cache
+        @lock_cache ||= {}
+      end
+
+      def lock_cache=(obj)
+        @lock_cache = obj
+      end
+    end
   end
 end

data/lib/sidekiq_unique_jobs/run_lock_failed.rb

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 SidekiqUniqueJobs::RunLockFailed = Class.new(StandardError)

data/lib/sidekiq_unique_jobs/script_mock.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pathname'
 require 'digest/sha1'
 
@@ -13,53 +15,52 @@ module SidekiqUniqueJobs
     end
 
     def acquire_lock(redis_pool, options = {})
-      connection(redis_pool) do |redis|
+      connection(redis_pool) do |conn|
         unique_key = options[:keys][0]
         job_id     = options[:argv][0]
         expires    = options[:argv][1].to_i
-        stored_jid = redis.get(unique_key)
+        stored_jid = conn.get(unique_key)
 
         return (stored_jid == job_id) ? 1 : 0 if stored_jid
+        return 0 unless conn.set(unique_key, job_id, nx: true, ex: expires)
+
+        conn.hsetnx(SidekiqUniqueJobs::HASH_KEY, job_id, unique_key)
 
-        return 0 unless redis.set(unique_key, job_id, nx: true, ex: expires)
-        redis.hsetnx(SidekiqUniqueJobs::HASH_KEY, job_id, unique_key)
         return 1
       end
     end
 
     def release_lock(redis_pool, options = {})
-      connection(redis_pool) do |redis|
+      connection(redis_pool) do |conn|
         unique_key = options[:keys][0]
         job_id     = options[:argv][0]
-        stored_jid = redis.get(unique_key)
+        stored_jid = conn.get(unique_key)
 
         return -1 unless stored_jid
         return 0 unless stored_jid == job_id || stored_jid == '2'
 
-        redis.del(unique_key)
-        redis.hdel(SidekiqUniqueJobs::HASH_KEY, job_id)
+        conn.del(unique_key)
+        conn.hdel(SidekiqUniqueJobs::HASH_KEY, job_id)
+
         return 1
       end
     end
 
     def synchronize(redis_pool, options = {})
-      connection(redis_pool) do |redis|
+      connection(redis_pool) do |conn|
         unique_key = options[:keys][0]
         time       = options[:argv][0].to_i
         expires    = options[:argv][1].to_f
 
-        return 1 if redis.set(unique_key, time + expires, nx: true, ex: expires)
+        return 1 if conn.set(unique_key, time + expires, nx: true, ex: expires)
 
-        stored_time = redis.get(unique_key)
+        stored_time = conn.get(unique_key)
         if stored_time && stored_time < time
-          if redis.set(unique_key, time + expires, xx: true, ex: expires)
-            return 1
-          end
+          return 1 if conn.set(unique_key, time + expires, xx: true, ex: expires)
         end
 
         return 0
       end
     end
   end
-  # rubocop:enable MethodLength
 end

data/lib/sidekiq_unique_jobs/scripts/acquire_lock.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
   module Scripts
     class AcquireLock

data/lib/sidekiq_unique_jobs/scripts/release_lock.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
   module Scripts
     class ReleaseLock

data/lib/sidekiq_unique_jobs/scripts.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pathname'
 require 'digest/sha1'
 require 'concurrent/map'
@@ -22,22 +24,25 @@ module SidekiqUniqueJobs
     extend SingleForwardable
     def_delegators :SidekiqUniqueJobs, :connection, :logger
 
-    def call(file_name, redis_pool, options = {}) # rubocop:disable MethodLength
-      connection(redis_pool) do |redis|
-        if SCRIPT_SHAS[file_name].nil?
-          SCRIPT_SHAS[file_name] = redis.script(:load, script_source(file_name))
-        end
-        redis.evalsha(SCRIPT_SHAS[file_name], options)
-      end
+    def call(file_name, redis_pool, options = {})
+      internal_call(file_name, redis_pool, options)
     rescue Redis::CommandError => ex
-      if ex.message == 'NOSCRIPT No matching script. Please use EVAL.'
+      handle_error(ex, file_name, redis_pool, options)
+    end
+
+    def internal_call(file_name, redis_pool, options = {})
+      connection(redis_pool) do |conn|
+        SCRIPT_SHAS[file_name] = conn.script(:load, script_source(file_name)) if SCRIPT_SHAS[file_name].nil?
+        conn.evalsha(SCRIPT_SHAS[file_name], options)
+      end
+    end
+
+    def handle_error(exception, file_name, redis_pool, options = {})
+      if exception.message == 'NOSCRIPT No matching script. Please use EVAL.' # rubocop:disable Style/GuardClause
         SCRIPT_SHAS.delete(file_name)
         call(file_name, redis_pool, options)
-        raise
       else
-        raise ScriptError,
-              "#{file_name}.lua\n\n#{ex.message}\n\n#{script_source(file_name)}" \
-              "#{ex.backtrace.join("\n")}"
+        raise ScriptError, "Problem compiling #{file_name}. Invalid LUA syntax?"
       end
     end
 

data/lib/sidekiq_unique_jobs/server/middleware.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
   module Server
     class Middleware

data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sidekiq/api'
 
 module Sidekiq
@@ -5,7 +7,6 @@ module Sidekiq
     module UniqueExtension
       def self.included(base)
         base.class_eval do
-          include SidekiqUniqueJobs::Unlockable
           alias_method :delete_orig, :delete
           alias_method :delete, :delete_ext
           alias_method :remove_job_orig, :remove_job
@@ -34,7 +35,6 @@ module Sidekiq
     module UniqueExtension
       def self.included(base)
         base.class_eval do
-          include SidekiqUniqueJobs::Unlockable
           alias_method :delete_orig, :delete
           alias_method :delete, :delete_ext
         end
@@ -43,13 +43,6 @@ module Sidekiq
       def delete_ext
         SidekiqUniqueJobs::Unlockable.unlock(item) if delete_orig
       end
-
-      def remove_job_ext
-        remove_job_orig do |message|
-          SidekiqUniqueJobs::Unlockable.unlock(Sidekiq.load_json(message))
-          yield message
-        end
-      end
     end
     include UniqueExtension
   end
@@ -58,15 +51,14 @@ module Sidekiq
     module UniqueExtension
       def self.included(base)
         base.class_eval do
-          include SidekiqUniqueJobs::Unlockable
           alias_method :delete_orig, :delete
           alias_method :delete, :delete_ext
         end
       end
 
       def delete_ext
-        SidekiqUniqueJobs::Unlockable.unlock(item)
         delete_orig
+        SidekiqUniqueJobs::Unlockable.unlock(item)
       end
     end
 
@@ -77,7 +69,6 @@ module Sidekiq
     module UniqueExtension
       def self.included(base)
         base.class_eval do
-          include SidekiqUniqueJobs::Unlockable
           alias_method :clear_orig, :clear
           alias_method :clear, :clear_ext
         end
@@ -96,7 +87,6 @@ module Sidekiq
     module UniqueExtension
       def self.included(base)
         base.class_eval do
-          include SidekiqUniqueJobs::Unlockable
           if base.method_defined?(:clear)
             alias_method :clear_orig, :clear
             alias_method :clear, :clear_ext
@@ -115,7 +105,7 @@ module Sidekiq
       end
 
       def delete_by_value_ext(name, value)
-        SidekiqUniqueJobs::Unlockable.unlock(JSON.parse(value)) if delete_by_value_orig(name, value)
+        SidekiqUniqueJobs::Unlockable.unlock(Sidekiq.load_json(value)) if delete_by_value_orig(name, value)
       end
     end
 

data/lib/sidekiq_unique_jobs/testing/sidekiq_overrides.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sidekiq/testing'
 
 module Sidekiq

data/lib/sidekiq_unique_jobs/testing.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sidekiq_unique_jobs/testing/sidekiq_overrides'
 require 'sidekiq_unique_jobs/script_mock'
 
@@ -44,8 +46,8 @@ module SidekiqUniqueJobs
         worker_class = SidekiqUniqueJobs.worker_class_constantize(worker_class)
 
         if Sidekiq::Testing.inline?
-          _server.call(worker_class.new, item, queue, redis_pool) do
-            call_real(worker_class, item, queue, redis_pool) do
+          call_real(worker_class, item, queue, redis_pool) do
+            _server.call(worker_class.new, item, queue, redis_pool) do
               yield
             end
           end

data/lib/sidekiq_unique_jobs/timeout_calculator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
   class TimeoutCalculator
     def self.for_item(item)

data/lib/sidekiq_unique_jobs/unique_args.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'digest'
 require 'sidekiq_unique_jobs/normalizer'
 
@@ -5,9 +7,8 @@ module SidekiqUniqueJobs
   # This class exists to be testable and the entire api should be considered private
   # rubocop:disable ClassLength
   class UniqueArgs
-    CLASS_NAME = 'SidekiqUniqueJobs::UniqueArgs'.freeze
+    CLASS_NAME = 'SidekiqUniqueJobs::UniqueArgs'
     extend Forwardable
-    include Normalizer
 
     def_delegators :SidekiqUniqueJobs, :config, :worker_class_constantize
     def_delegators :Sidekiq, :logger
@@ -16,9 +17,9 @@ module SidekiqUniqueJobs
       new(item).unique_digest
     end
 
-    def initialize(job)
+    def initialize(item)
       Sidekiq::Logging.with_context(CLASS_NAME) do
-        @item = job
+        @item = item
         @worker_class            ||= worker_class_constantize(@item[CLASS_KEY])
         @item[UNIQUE_PREFIX_KEY] ||= unique_prefix
         @item[UNIQUE_ARGS_KEY]     = unique_args(@item[ARGS_KEY])
@@ -41,15 +42,16 @@ module SidekiqUniqueJobs
     end
 
     def digestable_hash
-      hash = @item.slice(CLASS_KEY, QUEUE_KEY, UNIQUE_ARGS_KEY)
-
-      if unique_on_all_queues?
-        logger.debug do
-          "#{__method__} uniqueness specified across all queues (deleting queue: #{@item[QUEUE_KEY]} from hash)"
+      @item.slice(CLASS_KEY, QUEUE_KEY, UNIQUE_ARGS_KEY).tap do |hash|
+        if unique_on_all_queues?
+          logger.debug { "#{__method__} deleting queue: #{@item[QUEUE_KEY]}" }
+          hash.delete(QUEUE_KEY)
+        end
+        if unique_across_workers?
+          logger.debug { "#{__method__} deleting class: #{@item[CLASS_KEY]}" }
+          hash.delete(CLASS_KEY)
         end
-        hash.delete(QUEUE_KEY)
       end
-      hash
     end
 
     def unique_args(args)
@@ -59,9 +61,13 @@ module SidekiqUniqueJobs
         logger.debug { "#{__method__} : unique arguments disabled" }
         args
       end
-    rescue NameError
-      # fallback to not filtering args when class can't be instantiated
-      return args
+    rescue NameError => ex
+      logger.error "#{__method__}(#{args}) : failed with (#{ex.message})"
+      logger.error ex
+
+      raise if config.raise_unique_args_errors
+
+      args
     end
 
     def unique_on_all_queues?
@@ -69,6 +75,11 @@ module SidekiqUniqueJobs
       @item[UNIQUE_ON_ALL_QUEUES_KEY] || @worker_class.get_sidekiq_options[UNIQUE_ON_ALL_QUEUES_KEY]
     end
 
+    def unique_across_workers?
+      return unless sidekiq_worker_class?
+      @item[UNIQUE_ACROSS_WORKERS_KEY] || @worker_class.get_sidekiq_options[UNIQUE_ACROSS_WORKERS_KEY]
+    end
+
     def unique_args_enabled?
       return unless sidekiq_worker_class?
       unique_args_method # && !unique_args_method.is_a?(Boolean)
@@ -106,6 +117,7 @@ module SidekiqUniqueJobs
         logger.warn { "#{__method__} : unique_args_method is nil. Returning (#{args})" }
         return args
       end
+
       filter_args = unique_args_method.call(args)
       logger.debug { "#{__method__} : #{args} -> #{filter_args}" }
       filter_args
@@ -134,4 +146,5 @@ module SidekiqUniqueJobs
       @unique_args_method ||= Sidekiq.default_worker_options.stringify_keys[UNIQUE_ARGS_KEY]
     end
   end
+  # rubocop:enable ClassLength
 end

data/lib/sidekiq_unique_jobs/unlockable.rb

@@ -1,22 +1,22 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
   module Unlockable
     module_function
 
     def unlock(item)
+      return unless item[UNIQUE_DIGEST_KEY]
       unlock_by_key(item[UNIQUE_DIGEST_KEY], item[JID_KEY])
     end
 
     def unlock_by_key(unique_key, jid, redis_pool = nil)
-      return false unless Scripts::ReleaseLock.execute(redis_pool, unique_key, jid)
-      after_unlock(jid)
-    end
-
-    def after_unlock(jid)
+      lock_released = Scripts::ReleaseLock.execute(redis_pool, unique_key, jid)
       ensure_job_id_removed(jid)
+      lock_released
     end
 
     def ensure_job_id_removed(jid)
-      Sidekiq.redis { |redis| redis.hdel(SidekiqUniqueJobs::HASH_KEY, jid) }
+      Sidekiq.redis { |conn| conn.hdel(SidekiqUniqueJobs::HASH_KEY, jid) }
     end
 
     def logger

data/lib/sidekiq_unique_jobs/util.rb

@@ -1,19 +1,24 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
-  module Util
-    SCAN_PATTERN ||= '*'.freeze
-    DEFAULT_COUNT ||= 1_000
-    KEYS_METHOD ||= 'keys'.freeze
-    SCAN_METHOD ||= 'scan'.freeze
+  module Util # rubocop:disable Metrics/ModuleLength
+    COUNT             = 'COUNT'
+    DEFAULT_COUNT     = 1_000
+    EXPIRE_BATCH_SIZE = 100
+    MATCH             = 'MATCH'
+    KEYS_METHOD       = 'keys'
+    SCAN_METHOD       = 'scan'
+    SCAN_PATTERN      = '*'
 
-    module_function
+    extend self # rubocop:disable Style/ModuleFunction
 
     def keys(pattern = SCAN_PATTERN, count = DEFAULT_COUNT)
       send("keys_by_#{redis_keys_method}", pattern, count)
     end
 
     def unique_key(jid)
-      connection do |redis|
-        redis.hget(SidekiqUniqueJobs::HASH_KEY, jid)
+      connection do |conn|
+        conn.hget(SidekiqUniqueJobs::HASH_KEY, jid)
       end
     end
 
@@ -23,7 +28,7 @@ module SidekiqUniqueJobs
       keys, time = timed { keys(pattern, count) }
       logger.debug { "#{keys.size} matching keys found in #{time} sec." }
       keys = dry_run(keys)
-      logger.debug { "#{keys.size} matching keys after postprocessing" }
+      logger.debug { "#{keys.size} matching keys after post-processing" }
       unless dry_run
         logger.debug { "deleting #{keys}..." }
         _, time = timed { batch_delete(keys) }
@@ -38,18 +43,33 @@ module SidekiqUniqueJobs
       end
     end
 
-    def expire
+    def expire # rubocop:disable Metrics/MethodLength
       removed_keys = {}
       connection do |conn|
-        conn.hgetall(SidekiqUniqueJobs::HASH_KEY).each do |jid, unique_key|
-          next if conn.get(unique_key)
-          conn.hdel(SidekiqUniqueJobs::HASH_KEY, jid)
-          removed_keys[jid] = unique_key
+        cursor = '0'
+        loop do
+          cursor, jobs = get_jobs(conn, cursor)
+          jobs.each do |job_array|
+            jid, unique_key = job_array
+
+            next if conn.get(unique_key)
+            conn.hdel(SidekiqUniqueJobs::HASH_KEY, jid)
+            removed_keys[jid] = unique_key
+          end
+
+          break if cursor == '0'
         end
       end
+
       removed_keys
     end
 
+    private
+
+    def get_jobs(conn, cursor)
+      conn.hscan(SidekiqUniqueJobs::HASH_KEY, [cursor, MATCH, SCAN_PATTERN, COUNT, EXPIRE_BATCH_SIZE])
+    end
+
     def keys_by_scan(pattern, count)
       connection { |conn| conn.scan_each(match: prefix(pattern), count: count).to_a }
     end

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
-  VERSION = '5.0.2'.freeze
+  VERSION = '5.0.11'
 end

data/redis/acquire_lock.lua

@@ -12,7 +12,7 @@ if stored_jid then
 end
 
 if redis.pcall('set', unique_key, job_id, 'nx', 'ex', expires) then
-  redis.pcall('hsetnx', 'uniquejobs', job_id, unique_key)
+  -- redis.pcall('hsetnx', 'uniquejobs', job_id, unique_key)
   return 1
 else
   return 0