sidekiq-unique-jobs 3.0.11 → 8.0.10

data/bin/uniquejobs

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "sidekiq_unique_jobs"
+
+SidekiqUniqueJobs::Cli.start(ARGV)

data/lib/sidekiq-unique-jobs.rb

@@ -1,37 +1,3 @@
-# rubocop:disable FileName
-require 'yaml' if RUBY_VERSION.include?('2.0.0')
-require 'sidekiq_unique_jobs/middleware'
-require 'sidekiq_unique_jobs/version'
-require 'sidekiq_unique_jobs/config'
-require 'sidekiq_unique_jobs/payload_helper'
-require 'ostruct'
+# frozen_string_literal: true
 
-module SidekiqUniqueJobs
-  module_function
-
-  def config
-    @config ||= Config.new(
-      unique_prefix: 'sidekiq_unique',
-      unique_args_enabled: false,
-      default_expiration: 30 * 60,
-      default_unlock_order: :after_yield
-    )
-  end
-
-  def unique_args_enabled?
-    config.unique_args_enabled
-  end
-
-  def configure
-    yield config
-  end
-
-  # Attempt to constantize a string worker_class argument, always
-  # failing back to the original argument.
-  def worker_class_constantize(worker_class)
-    return worker_class unless worker_class.is_a?(String)
-    worker_class.constantize
-  rescue NameError
-    worker_class
-  end
-end
+require "sidekiq_unique_jobs"

data/lib/sidekiq_unique_jobs/batch_delete.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class BatchDelete provides batch deletion of digests
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class BatchDelete
+    #
+    # @return [Integer] the default batch size
+    BATCH_SIZE = 500
+
+    #
+    # @return [Array<String>] Supported key suffixes
+    SUFFIXES = %w[
+      QUEUED
+      PRIMED
+      LOCKED
+      INFO
+    ].freeze
+
+    # includes "SidekiqUniqueJobs::Connection"
+    # @!parse include SidekiqUniqueJobs::Connection
+    include SidekiqUniqueJobs::Connection
+    # includes "SidekiqUniqueJobs::Logging"
+    # @!parse include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Logging
+
+    #
+    # @!attribute [r] digests
+    #   @return [Array<String>] a collection of digests to be deleted
+    attr_reader :digests
+    #
+    # @!attribute [r] conn
+    #   @return [Redis, RedisConnection, ConnectionPool] a redis connection
+    attr_reader :conn
+
+    #
+    # Executes a batch deletion of the provided digests
+    #
+    # @param [Array<String>] digests the digests to delete
+    # @param [Redis] conn the connection to use for deletion
+    #
+    # @return [void]
+    #
+    def self.call(digests, conn = nil)
+      new(digests, conn).call
+    end
+
+    #
+    # Initialize a new batch delete instance
+    #
+    # @param [Array<String>] digests the digests to delete
+    # @param [Redis] conn the connection to use for deletion
+    #
+    def initialize(digests, conn)
+      @count   = 0
+      @digests = digests
+      @conn    = conn
+      @digests ||= []
+      @digests.compact!
+      redis_version # Avoid pipelined calling redis_version and getting a future.
+    end
+
+    #
+    # Executes a batch deletion of the provided digests
+    # @note Just wraps batch_delete to be able to provide no connection
+    #
+    #
+    def call
+      return log_info("Nothing to delete; exiting.") if digests.none?
+
+      log_info("Deleting batch with #{digests.size} digests")
+      return batch_delete(conn) if conn
+
+      redis { |rcon| batch_delete(rcon) }
+    end
+
+    private
+
+    #
+    # Does the actual batch deletion
+    #
+    #
+    # @return [Integer] the number of deleted digests
+    #
+    def batch_delete(conn)
+      digests.each_slice(BATCH_SIZE) do |chunk|
+        conn.pipelined do |pipeline|
+          chunk.each do |digest|
+            del_digest(pipeline, digest)
+            pipeline.zrem(SidekiqUniqueJobs::DIGESTS, digest)
+            pipeline.zrem(SidekiqUniqueJobs::EXPIRING_DIGESTS, digest)
+            @count += 1
+          end
+        end
+      end
+
+      @count
+    end
+
+    def del_digest(pipeline, digest)
+      removable_keys = keys_for_digest(digest)
+
+      pipeline.unlink(*removable_keys)
+    end
+
+    def keys_for_digest(digest)
+      [digest, "#{digest}:RUN"].each_with_object([]) do |key, digest_keys|
+        digest_keys.push(key)
+        digest_keys.concat(SUFFIXES.map { |suffix| "#{key}:#{suffix}" })
+      end
+    end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/changelog.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Changelogs provides access to the changelog entries
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Changelog < Redis::SortedSet
+    def initialize
+      super(CHANGELOGS)
+    end
+
+    #
+    # Adds a new changelog entry
+    #
+    # @param [String] message a descriptive message about the entry
+    # @param [String] digest a unique digest
+    # @param [String] job_id a Sidekiq JID
+    # @param [String] script the name of the script adding the entry
+    #
+    # @return [void]
+    #
+    def add(message:, digest:, job_id:, script:)
+      message = dump_json(message: message, digest: digest, job_id: job_id, script: script)
+      redis { |conn| conn.zadd(key, now_f, message) }
+    end
+
+    #
+    # The change log entries
+    #
+    # @param [String] pattern the pattern to match
+    # @param [Integer] count the number of matches to return
+    #
+    # @return [Array<Hash>] an array of entries
+    #
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
+      redis do |conn|
+        conn.zscan(key, match: pattern, count: count).to_a.map { |entry| load_json(entry[0]) }
+      end
+    end
+
+    #
+    # Paginate the changelog entries
+    #
+    # @param [Integer] cursor the cursor for this iteration
+    # @param [String] pattern "*" the pattern to match
+    # @param [Integer] page_size 100 the number of matches to return
+    #
+    # @return [Array<Integer, Integer, Array<Hash>] the total size, next cursor and changelog entries
+    #
+    def page(cursor: 0, pattern: "*", page_size: 100)
+      redis do |conn|
+        total_size, result = conn.multi do |pipeline|
+          pipeline.zcard(key)
+          pipeline.zscan(key, cursor, match: pattern, count: page_size)
+        end
+
+        # NOTE: When debugging, check the last item in the returned array.
+        [
+          total_size.to_i,
+          result[0].to_i, # next_cursor
+          result[1].map { |entry| load_json(entry) }.select { |entry| entry.is_a?(Hash) },
+        ]
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/cli.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "thor"
+
+module SidekiqUniqueJobs
+  #
+  # Command line interface for unique jobs
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Cli < Thor
+    # :nodoc:
+    # rubocop:disable Style/OptionalBooleanParameter
+    def self.banner(command, _namespace = nil, _subcommand = false) # rubocop:disable Style/OptionalBooleanParameter
+      "jobs #{@package_name} #{command.usage}" # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
+    end
+    # rubocop:enable Style/OptionalBooleanParameter
+
+    desc "list PATTERN", "list all unique digests and their expiry time"
+    option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
+    # :nodoc:
+    def list(pattern = "*")
+      max_count = options[:count]
+      say "Searching for regular digests"
+      list_entries(digests.entries(pattern: pattern, count: max_count), pattern)
+      say "Searching for expiring digests"
+      list_entries(expiring_digests.entries(pattern: pattern, count: max_count), pattern)
+    end
+
+    desc "del PATTERN", "deletes unique digests from redis by pattern"
+    option :dry_run, aliases: :d, type: :boolean, desc: "set to false to perform deletion"
+    option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
+    # :nodoc:
+    def del(pattern)
+      max_count = options[:count]
+      if options[:dry_run]
+        count_entries_for_del(max_count, pattern)
+      else
+        del_entries(max_count, pattern)
+      end
+    end
+
+    desc "console", "drop into a console with easy access to helper methods"
+    # :nodoc:
+    def console
+      say "Use `list '*', 1000 to display the first 1000 unique digests matching '*'"
+      say "Use `del '*', 1000, true (default) to see how many digests would be deleted for the pattern '*'"
+      say "Use `del '*', 1000, false to delete the first 1000 digests matching '*'"
+
+      # Object.include SidekiqUniqueJobs::Api
+      console_class.start
+    end
+
+    no_commands do # rubocop:disable Metrics/BlockLength
+      # :nodoc:
+      def digests
+        @digests ||= SidekiqUniqueJobs::Digests.new
+      end
+
+      # :nodoc:
+      def expiring_digests
+        @expiring_digests ||= SidekiqUniqueJobs::ExpiringDigests.new
+      end
+
+      # :nodoc:
+      def console_class
+        require "pry"
+        Pry
+      rescue NameError, LoadError
+        require "irb"
+        IRB
+      end
+
+      # :nodoc:
+      def list_entries(entries, pattern)
+        say "Found #{entries.size} digests matching '#{pattern}':"
+        print_in_columns(entries.sort) if entries.any?
+      end
+
+      # :nodoc:
+      def count_entries_for_del(max_count, pattern)
+        count = digests.entries(pattern: pattern, count: max_count).size +
+                expiring_digests.entries(pattern: pattern, count: max_count).size
+        say "Would delete #{count} digests matching '#{pattern}'"
+      end
+
+      # :nodoc:
+      def del_entries(max_count, pattern)
+        deleted_count = digests.delete_by_pattern(pattern, count: max_count).to_i +
+                        expiring_digests.delete_by_pattern(pattern, count: max_count).to_i
+        say "Deleted #{deleted_count} digests matching '#{pattern}'"
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/config.rb

@@ -1,46 +1,319 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
-  class Config < OpenStruct
-    CONFIG_ACCESSORS = [
-      :unique_prefix,
-      :unique_args_enabled,
-      :default_expiration,
-      :default_unlock_order
-    ]
+  # ThreadSafe config exists to be able to document the config class without errors
+  ThreadSafeConfig = Concurrent::MutableStruct.new("ThreadSafeConfig",
+                                                   :lock_timeout,
+                                                   :lock_ttl,
+                                                   :enabled,
+                                                   :lock_prefix,
+                                                   :logger,
+                                                   :logger_enabled,
+                                                   :locks,
+                                                   :strategies,
+                                                   :debug_lua,
+                                                   :max_history,
+                                                   :reaper,
+                                                   :reaper_count,
+                                                   :reaper_interval,
+                                                   :reaper_timeout,
+                                                   :reaper_resurrector_interval,
+                                                   :reaper_resurrector_enabled,
+                                                   :lock_info,
+                                                   :raise_on_config_error,
+                                                   :current_redis_version)
+
+  #
+  # Shared class for dealing with gem configuration
+  #
+  # @author Mauro Berlanda <mauro.berlanda@gmail.com>
+  # rubocop:disable Metrics/ClassLength
+  class Config < ThreadSafeConfig
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available queued locks
+    LOCKS_WHILE_ENQUEUED = {
+      until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
+      while_enqueued: SidekiqUniqueJobs::Lock::UntilExecuting,
+    }.freeze
+
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available fulltime locks
+    LOCKS_FROM_PUSH_TO_PROCESSED = {
+      until_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_performed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_processed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
+      until_successfully_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    }.freeze
+
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available locks without unlock
+    LOCKS_WITHOUT_UNLOCK = {
+      until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
+    }.freeze
+
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available runtime/client locks
+    LOCKS_WHEN_BUSY = {
+      around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_busy: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_working: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
+    }.freeze
+
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available default locks
+    LOCKS =
+      LOCKS_WHEN_BUSY.dup
+        .merge(LOCKS_WHILE_ENQUEUED.dup)
+        .merge(LOCKS_WITHOUT_UNLOCK.dup)
+        .merge(LOCKS_FROM_PUSH_TO_PROCESSED.dup)
+        .freeze
+
+    #
+    # @return [Hash<Symbol, SidekiqUniqueJobs::OnConflict::Strategy] all available default strategies
+    STRATEGIES = {
+      log: SidekiqUniqueJobs::OnConflict::Log,
+      raise: SidekiqUniqueJobs::OnConflict::Raise,
+      reject: SidekiqUniqueJobs::OnConflict::Reject,
+      replace: SidekiqUniqueJobs::OnConflict::Replace,
+      reschedule: SidekiqUniqueJobs::OnConflict::Reschedule,
+    }.freeze
+
+    #
+    # @return ['uniquejobs'] by default we use this prefix
+    PREFIX                = "uniquejobs"
+    #
+    # @return [0] by default don't wait for locks
+    LOCK_TIMEOUT          = 0
+    #
+    # @return [nil]
+    LOCK_TTL              = nil
+    #
+    # @return [true,false] by default false (don't disable logger)
+    LOGGER_ENABLED        = true
+    #
+    # @return [true] by default the gem is enabled
+    ENABLED               = true
+    #
+    # @return [false] by default we don't debug the lua scripts because it is slow
+    DEBUG_LUA             = false
+    #
+    # @return [1_000] use a changelog history of 1_000 entries by default
+    MAX_HISTORY           = 1_000
+    #
+    # @return [:ruby] prefer the ruby reaper by default since the lua reaper still has problems
+    REAPER                = :ruby
+    #
+    # @return [1_000] reap 1_000 orphaned locks at a time by default
+    REAPER_COUNT          = 1_000
+    #
+    # @return [600] reap locks every 10 minutes
+    REAPER_INTERVAL       = 600
+    #
+    # @return [10] stop reaper after 10 seconds
+    REAPER_TIMEOUT        = 10
+    #
+    # @return [3600] check if reaper is dead each 3600 seconds
+    REAPER_RESURRECTOR_INTERVAL = 3600
+
+    #
+    # @return [false] enable reaper resurrector
+    REAPER_RESURRECTOR_ENABLED = false
+
+    #
+    # @return [false] while useful it also adds overhead so disable lock_info by default
+    USE_LOCK_INFO         = false
+    #
+    # @return [false] by default we don't raise validation errors for workers
+    RAISE_ON_CONFIG_ERROR = false
+    #
+    # @return [0.0.0] default redis version is only to avoid NoMethodError on nil
+    REDIS_VERSION         = "0.0.0"
 
-    class << self
-      CONFIG_ACCESSORS.each do |method|
-        define_method(method) do
-          warn("#{method} has been deprecated. See readme for information")
-          config.send(method)
-        end
+    #
+    # Returns a default configuration
+    #
+    # @example
+    #   SidekiqUniqueJobs::Config.default => <concurrent/mutable_struct/thread_safe_config SidekiqUniqueJobs::Config {
+    #   default_lock_timeout: 0,
+    #   default_lock_ttl: nil,
+    #   enabled: true,
+    #   lock_prefix: "uniquejobs",
+    #   logger: #<Sidekiq::Logger:0x00007f81e096b0e0 @level=1 ...>,
+    #   locks: {
+    #     around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_busy: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_working: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
+    #     until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
+    #     while_enqueued: SidekiqUniqueJobs::Lock::UntilExecuting,
+    #     until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
+    #     until_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_performed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_processed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
+    #     until_successfully_completed: SidekiqUniqueJobs::Lock::UntilExecuted
+    #   },
+    #   strategies: {
+    #     log: SidekiqUniqueJobs::OnConflict::Log,
+    #     raise: SidekiqUniqueJobs::OnConflict::Raise,
+    #     reject: SidekiqUniqueJobs::OnConflict::Reject,
+    #     replace: SidekiqUniqueJobs::OnConflict::Replace,
+    #     reschedule: SidekiqUniqueJobs::OnConflict::Reschedule
+    #   },
+    #   debug_lua: false,
+    #   max_history: 1000,
+    #   reaper:: ruby,
+    #   reaper_count: 1000,
+    #   lock_info: false,
+    #   raise_on_config_error: false,
+    #   }>
+    #
+    #
+    # @return [SidekiqUniqueJobs::Config] a default configuration
+    #
+    def self.default # rubocop:disable Metrics/MethodLength
+      new(
+        LOCK_TIMEOUT,
+        LOCK_TTL,
+        ENABLED,
+        PREFIX,
+        Sidekiq.logger,
+        LOGGER_ENABLED,
+        LOCKS,
+        STRATEGIES,
+        DEBUG_LUA,
+        MAX_HISTORY,
+        REAPER,
+        REAPER_COUNT,
+        REAPER_INTERVAL,
+        REAPER_TIMEOUT,
+        REAPER_RESURRECTOR_INTERVAL,
+        REAPER_RESURRECTOR_ENABLED,
+        USE_LOCK_INFO,
+        RAISE_ON_CONFIG_ERROR,
+        REDIS_VERSION,
+      )
+    end
+
+    #
+    # Set the default_lock_ttl
+    # @deprecated
+    #
+    # @param [Integer] obj value to set (seconds)
+    #
+    # @return [<type>] <description>
+    #
+    def default_lock_ttl=(obj)
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_ttl=` instead."
+      self.lock_ttl = obj
+    end
+
+    #
+    # Set new value for default_lock_timeout
+    # @deprecated
+    #
+    # @param [Integer] obj value to set (seconds)
+    #
+    # @return [Integer]
+    #
+    def default_lock_timeout=(obj)
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_timeout=` instead."
+      self.lock_timeout = obj
+    end
 
-        define_method("#{method}=") do |obj|
-          warn("#{method} has been deprecated. See readme for information")
-          config.send("#{method}=", obj)
-        end
-      end
+    #
+    # Default lock TTL (Time To Live)
+    # @deprecated
+    #
+    # @return [nil, Integer] configured value or nil
+    #
+    def default_lock_ttl
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_ttl` instead."
+      lock_ttl
+    end
+
+    #
+    # Default Lock Timeout
+    # @deprecated
+    #
+    #
+    # @return [nil, Integer] configured value or nil
+    #
+    def default_lock_timeout
+      warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
+           " Please use `#{class_name}#lock_timeout` instead."
+      lock_timeout
+    end
+
+    #
+    # Memoized variable to get the class name
+    #
+    #
+    # @return [String] name of the class
+    #
+    def class_name
+      @class_name ||= self.class.name
+    end
 
-      def unique_args_enabled?
-        warn('unique_args_enabled has been deprecated. See readme for information')
-        config.unique_args_enabled
-      end
+    #
+    # Adds a lock type to the configuration. It will raise if the lock exists already
+    #
+    # @example Add a custom lock
+    #   add_lock(:my_lock, CustomLocks::MyLock)
+    #
+    # @raise DuplicateLock when the name already exists
+    #
+    # @param [String, Symbol] name the name of the lock
+    # @param [Class] klass the class describing the lock
+    #
+    # @return [void]
+    #
+    def add_lock(name, klass)
+      lock_sym = name.to_sym
+      raise DuplicateLock, ":#{name} already defined, please use another name" if locks.key?(lock_sym)
 
-      def config
-        SidekiqUniqueJobs.config
-      end
+      new_locks = locks.dup.merge(lock_sym => klass).freeze
+      self.locks = new_locks
     end
 
-    def testing_enabled?
-      if Sidekiq.const_defined?('Testing') && Sidekiq::Testing.enabled?
-        require 'sidekiq_unique_jobs/testing'
-        return true
-      end
+    #
+    # Adds an on_conflict strategy to the configuration.
+    #
+    # @example Add a custom strategy
+    #   add_lock(:my_strategy, CustomStrategies::MyStrategy)
+    #
+    # @raise [DuplicateStrategy] when the name already exists
+    #
+    # @param [String] name the name of the custom strategy
+    # @param [Class] klass the class describing the strategy
+    #
+    def add_strategy(name, klass)
+      strategy_sym = name.to_sym
+      raise DuplicateStrategy, ":#{name} already defined, please use another name" if strategies.key?(strategy_sym)
 
-      false
+      new_strategies = strategies.dup.merge(strategy_sym => klass).freeze
+      self.strategies = new_strategies
     end
 
-    def unique_args_enabled?
-      config.unique_args_enabled
+    #
+    # The current version of redis
+    #
+    #
+    # @return [String] a version string eg. `5.0.1`
+    #
+    def redis_version
+      self.current_redis_version = SidekiqUniqueJobs.fetch_redis_version if current_redis_version == REDIS_VERSION
+      current_redis_version
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end