sidekiq-unique-jobs 5.0.11 → 6.0.0.rc1

data/lib/sidekiq/simulator.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
+require 'sidekiq/cli'
 require 'sidekiq/launcher'
-require 'timeout'
+
+require 'sidekiq_unique_jobs/timeout'
 
 module Sidekiq
   class Simulator
-    extend Forwardable
-    def_delegator SidekiqUniqueJobs, :logger
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Timeout
 
     attr_reader :queues, :launcher
 
@@ -15,12 +17,13 @@ module Sidekiq
     end
 
     def initialize(queue)
-      @queues = [queue].flatten.uniq
+      @queues = Array(queue).uniq
       @launcher = Sidekiq::Launcher.new(sidekiq_options(queues))
     end
 
     def process_queue
-      run_launcher { yield }
+      run_launcher
+      yield
     ensure
       terminate_launcher
     end
@@ -28,15 +31,17 @@ module Sidekiq
     private
 
     def run_launcher
+      run_launcher!
+    rescue Timeout::Error => exception
+      log_warn('Timeout while starting Sidekiq')
+      log_warn(exception)
+    end
+
+    def run_launcher!
       using_timeout(15) do
         launcher.run
         sleep 0.001 until alive?
       end
-    rescue Timeout::Error => e
-      logger.warn { "Timeout while running #{__method__}" }
-      logger.warn { e }
-    ensure
-      yield
     end
 
     def terminate_launcher
@@ -51,12 +56,6 @@ module Sidekiq
       !alive?
     end
 
-    def using_timeout(value)
-      Timeout.timeout(value) do
-        yield
-      end
-    end
-
     def sidekiq_options(queues = [])
       { queues: queues,
         concurrency: 3,

data/lib/sidekiq_unique_jobs.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'yaml' if RUBY_VERSION.include?('2.0.0')
+require 'forwardable'
+require 'concurrent/mutable_struct'
+require 'ostruct'
+
+require 'sidekiq_unique_jobs/version'
+require 'sidekiq_unique_jobs/constants'
+require 'sidekiq_unique_jobs/logging'
+require 'sidekiq_unique_jobs/sidekiq_worker_methods'
+require 'sidekiq_unique_jobs/connection'
+require 'sidekiq_unique_jobs/exceptions'
+require 'sidekiq_unique_jobs/util'
+require 'sidekiq_unique_jobs/cli'
+require 'sidekiq_unique_jobs/core_ext'
+require 'sidekiq_unique_jobs/timeout'
+require 'sidekiq_unique_jobs/scripts'
+require 'sidekiq_unique_jobs/unique_args'
+require 'sidekiq_unique_jobs/unlockable'
+require 'sidekiq_unique_jobs/locksmith'
+require 'sidekiq_unique_jobs/options_with_fallback'
+require 'sidekiq_unique_jobs/middleware'
+require 'sidekiq_unique_jobs/sidekiq_unique_ext'
+
+module SidekiqUniqueJobs
+  include SidekiqUniqueJobs::Connection
+
+  module_function
+
+  Concurrent::MutableStruct.new(
+    'Config',
+    :default_lock_timeout,
+    :enabled,
+    :unique_prefix,
+    :logger,
+  )
+
+  def config
+    # Arguments here need to match the definition of the new class (see above)
+    @config ||= Concurrent::MutableStruct::Config.new(
+      0,
+      true,
+      'uniquejobs',
+      Sidekiq.logger,
+    )
+  end
+
+  def logger
+    config.logger
+  end
+
+  def logger=(other)
+    config.logger = other
+  end
+
+  def use_config(tmp_config)
+    fail ::ArgumentError, "#{name}.#{__method__} needs a block" unless block_given?
+
+    old_config = config.to_h
+    configure(tmp_config)
+    yield
+    configure(old_config)
+  end
+
+  def configure(options = {})
+    if block_given?
+      yield config
+    else
+      options.each do |key, val|
+        config.send("#{key}=", val)
+      end
+    end
+  end
+
+  def redis_version
+    @redis_version ||= redis { |conn| conn.info('server')['redis_version'] }
+  end
+end

data/lib/sidekiq_unique_jobs/cli.rb

@@ -4,17 +4,13 @@ require 'thor'
 
 module SidekiqUniqueJobs
   class Cli < Thor
-    # def initialize(argv, stdin = STDIN, stdout = STDOUT, stderr = STDERR, kernel = Kernel)
-    #   @argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
-    # end
-
     def self.banner(command, _namespace = nil, _subcommand = false)
       "jobs #{@package_name} #{command.usage}"
     end
 
     desc 'keys PATTERN', 'list all unique keys and their expiry time'
     option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
-    def keys(pattern)
+    def keys(pattern = '*')
       keys = Util.keys(pattern, options[:count])
       say "Found #{keys.size} keys matching '#{pattern}':"
       print_in_columns(keys.sort) if keys.any?
@@ -24,15 +20,14 @@ module SidekiqUniqueJobs
     option :dry_run, aliases: :d, type: :boolean, desc: 'set to false to perform deletion'
     option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
     def del(pattern)
-      deleted_count = Util.del(pattern, options[:count], options[:dry_run])
-      say "Deleted #{deleted_count} keys matching '#{pattern}'"
-    end
-
-    desc 'expire', 'removes all expired unique keys from the hash in redis'
-    def expire
-      expired = Util.expire
-      say "Removed #{expired.values.size} left overs from redis."
-      print_in_columns(expired.values)
+      max_count = options[:count]
+      if options[:dry_run]
+        keys = Util.keys(pattern, max_count)
+        say "Would delete #{keys.size} keys matching '#{pattern}'"
+      else
+        deleted_count = Util.del(pattern, max_count)
+        say "Deleted #{deleted_count} keys matching '#{pattern}'"
+      end
     end
 
     desc 'console', 'drop into a console with easy access to helper methods'
@@ -45,10 +40,6 @@ module SidekiqUniqueJobs
     end
 
     no_commands do
-      def logger
-        SidekiqUniqueJobs.logger
-      end
-
       def console_class
         require 'pry'
         Pry

data/lib/sidekiq_unique_jobs/client/middleware.rb

@@ -5,38 +5,36 @@ require 'sidekiq_unique_jobs/server/middleware'
 module SidekiqUniqueJobs
   module Client
     class Middleware
-      extend Forwardable
-      def_delegators :SidekiqUniqueJobs, :connection, :config, :worker_class_constantize
-      def_delegators :Sidekiq, :logger
-
+      include SidekiqUniqueJobs::Logging
       include OptionsWithFallback
 
+      # :reek:LongParameterList { max_params: 4 }
       def call(worker_class, item, queue, redis_pool = nil)
-        @worker_class = worker_class_constantize(worker_class)
-        @item = item
-        @queue = queue
-        @redis_pool = redis_pool
+        @worker_class = worker_class
+        @item         = item
+        @queue        = queue
+        @redis_pool   = redis_pool
 
-        yield if disabled_or_successfully_locked?
+        yield if success?
       end
 
       private
 
-      attr_reader :item, :worker_class, :redis_pool, :queue
+      attr_reader :item
 
-      def disabled_or_successfully_locked?
-        unique_disabled? || acquire_lock
+      def success?
+        unique_disabled? || locked?
       end
 
-      def acquire_lock
-        return true unless lock.respond_to?(:lock)
-        locked = lock.lock(:client)
-        warn_about_duplicate(item) unless locked
+      def locked?
+        locked = lock.lock
+        warn_about_duplicate unless locked
         locked
       end
 
-      def warn_about_duplicate(item)
-        logger.warn "payload is not unique #{item}" if log_duplicate_payload?
+      def warn_about_duplicate
+        return unless log_duplicate_payload?
+        log_warn "payload is not unique #{item}"
       end
     end
   end

data/lib/sidekiq_unique_jobs/connection.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Connection
+    def self.included(base)
+      base.send(:extend, self)
+    end
+
+    # :reek:UtilityFunction { enabled: false }
+    def redis(redis_pool = nil)
+      if redis_pool
+        redis_pool.with { |conn| yield conn }
+      else
+        Sidekiq.redis { |conn| yield conn }
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/constants.rb

@@ -1,22 +1,18 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  ARGS_KEY ||= 'args'
-  AT_KEY ||= 'at'
-  CLASS_KEY ||= 'class'
-  JID_KEY ||= 'jid'
-  LOG_DUPLICATE_KEY ||= 'log_duplicate_payload'
-  QUEUE_KEY ||= 'queue'
-  HASH_KEY ||= 'uniquejobs'
-  QUEUE_LOCK_TIMEOUT_KEY ||= 'unique_expiration'
-  RUN_LOCK_TIMEOUT_KEY ||= 'run_lock_expiration'
-  TESTING_CONSTANT ||= 'Testing'
-  UNIQUE_KEY ||= 'unique'
-  UNIQUE_LOCK_KEY ||= 'unique_lock'
-  UNIQUE_ARGS_KEY ||= 'unique_args'
-  UNIQUE_PREFIX_KEY ||= 'unique_prefix'
-  UNIQUE_DIGEST_KEY ||= 'unique_digest'
-  UNIQUE_ON_ALL_QUEUES_KEY ||= 'unique_on_all_queues'
+  ARGS_KEY                  ||= 'args'
+  AT_KEY                    ||= 'at'
+  CLASS_KEY                 ||= 'class'
+  JID_KEY                   ||= 'jid'
+  LOCK_EXPIRATION_KEY       ||= 'lock_expiration'
+  LOCK_TIMEOUT_KEY          ||= 'lock_timeout'
+  LOG_DUPLICATE_KEY         ||= 'log_duplicate_payload'
+  QUEUE_KEY                 ||= 'queue'
   UNIQUE_ACROSS_WORKERS_KEY ||= 'unique_across_workers'
-  UNIQUE_ARGS_ENABLED_KEY ||= 'unique_args_enabled'
+  UNIQUE_ARGS_KEY           ||= 'unique_args'
+  UNIQUE_DIGEST_KEY         ||= 'unique_digest'
+  UNIQUE_KEY                ||= 'unique'
+  UNIQUE_ON_ALL_QUEUES_KEY  ||= 'unique_on_all_queues'
+  UNIQUE_PREFIX_KEY         ||= 'unique_prefix'
 end

data/lib/sidekiq_unique_jobs/core_ext.rb

@@ -1,43 +1,40 @@
 # frozen_string_literal: true
 
-begin
-  require 'active_support/core_ext/hash/keys'
-  require 'active_support/core_ext/hash/deep_merge'
-rescue LoadError
-  class Hash
-    unless {}.respond_to?(:slice)
-      def slice(*keys)
-        keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
-        keys.each_with_object(self.class.new) { |k, hash| hash[k] = self[k] if key?(k) }
-      end
+# :nocov:
+
+class Hash
+  unless {}.respond_to?(:slice)
+    def slice(*keys)
+      keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
+      keys.each_with_object(self.class.new) { |k, hash| hash[k] = self[k] if key?(k) }
     end
+  end
 
-    unless {}.respond_to?(:stringify_keys)
-      def stringify_keys
-        transform_keys(&:to_s)
-      end
+  unless {}.respond_to?(:stringify_keys)
+    def stringify_keys
+      transform_keys(&:to_s)
     end
+  end
 
-    unless {}.respond_to?(:transform_keys)
-      def transform_keys
-        result = {}
-        each_key do |key|
-          result[yield(key)] = self[key]
-        end
-        result
+  unless {}.respond_to?(:transform_keys)
+    def transform_keys
+      result = {}
+      each_key do |key|
+        result[yield(key)] = self[key]
       end
+      result
     end
+  end
 
-    unless {}.respond_to?(:slice!)
-      def slice!(*keys)
-        keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
-        omit = slice(*self.keys - keys)
-        hash = slice(*keys)
-        hash.default      = default
-        hash.default_proc = default_proc if default_proc
-        replace(hash)
-        omit
-      end
+  unless {}.respond_to?(:slice!)
+    def slice!(*keys)
+      keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
+      omit = slice(*self.keys - keys)
+      hash = slice(*keys)
+      hash.default      = default
+      hash.default_proc = default_proc if default_proc
+      replace(hash)
+      omit
     end
   end
 end
@@ -53,27 +50,3 @@ class Array
     end
   end
 end
-
-class String
-  unless ''.respond_to?(:classify)
-    def classify
-      camelize(sub(/.*\./, ''))
-    end
-  end
-
-  unless ''.respond_to?(:camelize)
-    def camelize(uppercase_first_letter = true)
-      string = self
-      string = if uppercase_first_letter
-                 string.sub(/^[a-z\d]*/) { $&.capitalize }
-               else
-                 string.sub(/^(?:(?=\b|[A-Z_])|\w)/) { $&.downcase }
-               end
-      string.gsub!(%r{(?:_|(\/))([a-z\d]*)}i) do
-        "#{Regexp.last_match(1)}#{Regexp.last_match(2).capitalize}"
-      end
-      string.gsub!(%r{/}, '::')
-      string
-    end
-  end
-end

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class LockTimeout < StandardError
+  end
+
+  class RunLockFailed < StandardError
+  end
+
+  class ScriptError < StandardError
+    def initialize(file_name:, source_exception:)
+      super("Problem compiling #{file_name}. Message: #{source_exception.message}")
+    end
+  end
+
+  class UniqueKeyMissing < ArgumentError
+  end
+
+  class JidMissing < ArgumentError
+  end
+
+  class MaxLockTimeMissing < ArgumentError
+  end
+
+  class UnexpectedValue < StandardError
+  end
+
+  class UnknownLock < StandardError
+  end
+end

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    class BaseLock
+      include SidekiqUniqueJobs::Logging
+
+      def initialize(item, redis_pool = nil)
+        @item       = prepare_item(item)
+        @redis_pool = redis_pool
+      end
+
+      def lock
+        locksmith.lock(item[LOCK_TIMEOUT_KEY])
+      end
+
+      def execute(_callback = nil)
+        raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
+      end
+
+      def unlock
+        locksmith.signal(item[JID_KEY]) # Only signal to release the lock
+      end
+
+      def delete
+        locksmith.delete # Soft delete (don't forcefully remove when expiration is set)
+      end
+
+      def delete!
+        locksmith.delete! # Force delete the lock
+      end
+
+      def locked?
+        locksmith.locked?(item[JID_KEY])
+      end
+
+      private
+
+      attr_reader :item, :redis_pool, :operative
+
+      def locksmith
+        @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
+      end
+
+      def using_protection(callback)
+        @operative = true
+        yield
+      rescue Sidekiq::Shutdown
+        @operative = false
+        raise
+      ensure
+        unlock_and_callback(callback)
+      end
+
+      def prepare_item(item)
+        calculator = SidekiqUniqueJobs::Timeout::Calculator.new(item)
+        item[LOCK_TIMEOUT_KEY]    = calculator.lock_timeout
+        item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
+        SidekiqUniqueJobs::UniqueArgs.digest(item)
+        item
+      end
+
+      def unlock_and_callback(callback)
+        return notify_about_manual_unlock unless operative
+        unlock
+        delete
+
+        return notify_about_manual_unlock if locked?
+        callback_safely(callback)
+      end
+
+      def notify_about_manual_unlock
+        log_fatal("the unique_key: #{item[UNIQUE_DIGEST_KEY]} needs to be unlocked manually")
+      end
+
+      def callback_safely(callback)
+        callback.call
+      rescue StandardError
+        log_warn("the callback for unique_key: #{item[UNIQUE_DIGEST_KEY]} failed!")
+        raise
+      end
+    end
+  end
+end