sidekiq-unique-jobs 6.0.25 → 7.0.0.beta2

data/lib/sidekiq-unique-jobs.rb

@@ -1,3 +1,5 @@
 # frozen_string_literal: true
 
 require "sidekiq_unique_jobs"
+
+SidekiqUniqueJobs::Middleware.configure

data/lib/sidekiq_unique_jobs.rb

@@ -1,25 +1,56 @@
 # frozen_string_literal: true
 
-require "forwardable"
+require "concurrent/future"
+require "concurrent/promises"
+require "concurrent/timer_task"
+require "concurrent/map"
 require "concurrent/mutable_struct"
+require "digest"
+require "digest/sha1"
+require "erb"
+require "forwardable"
+require "json"
+require "pathname"
+require "sidekiq"
 
 require "sidekiq_unique_jobs/version"
 require "sidekiq_unique_jobs/version_check"
 require "sidekiq_unique_jobs/constants"
+require "sidekiq_unique_jobs/json"
 require "sidekiq_unique_jobs/logging"
+require "sidekiq_unique_jobs/logging/middleware_context"
+require "sidekiq_unique_jobs/timing"
 require "sidekiq_unique_jobs/sidekiq_worker_methods"
 require "sidekiq_unique_jobs/connection"
 require "sidekiq_unique_jobs/exceptions"
+require "sidekiq_unique_jobs/script"
+require "sidekiq_unique_jobs/script/caller"
+require "sidekiq_unique_jobs/script/template"
+require "sidekiq_unique_jobs/json"
+require "sidekiq_unique_jobs/normalizer"
 require "sidekiq_unique_jobs/job"
-require "sidekiq_unique_jobs/util"
-require "sidekiq_unique_jobs/digests"
+require "sidekiq_unique_jobs/redis"
+require "sidekiq_unique_jobs/redis/entity"
+require "sidekiq_unique_jobs/redis/hash"
+require "sidekiq_unique_jobs/redis/list"
+require "sidekiq_unique_jobs/redis/set"
+require "sidekiq_unique_jobs/redis/sorted_set"
+require "sidekiq_unique_jobs/redis/string"
+require "sidekiq_unique_jobs/batch_delete"
+require "sidekiq_unique_jobs/orphans/reaper"
+require "sidekiq_unique_jobs/orphans/observer"
+require "sidekiq_unique_jobs/orphans/manager"
 require "sidekiq_unique_jobs/cli"
 require "sidekiq_unique_jobs/core_ext"
-require "sidekiq_unique_jobs/timeout"
-require "sidekiq_unique_jobs/scripts"
+require "sidekiq_unique_jobs/time_calculator"
 require "sidekiq_unique_jobs/unique_args"
 require "sidekiq_unique_jobs/unlockable"
+require "sidekiq_unique_jobs/key"
 require "sidekiq_unique_jobs/locksmith"
+require "sidekiq_unique_jobs/options_with_fallback"
+require "sidekiq_unique_jobs/lock"
+require "sidekiq_unique_jobs/lock_config"
+require "sidekiq_unique_jobs/lock_info"
 require "sidekiq_unique_jobs/lock/base_lock"
 require "sidekiq_unique_jobs/lock/until_executed"
 require "sidekiq_unique_jobs/lock/until_executing"
@@ -27,9 +58,15 @@ require "sidekiq_unique_jobs/lock/until_expired"
 require "sidekiq_unique_jobs/lock/while_executing"
 require "sidekiq_unique_jobs/lock/while_executing_reject"
 require "sidekiq_unique_jobs/lock/until_and_while_executing"
-require "sidekiq_unique_jobs/options_with_fallback"
 require "sidekiq_unique_jobs/middleware"
+require "sidekiq_unique_jobs/middleware/client"
+require "sidekiq_unique_jobs/middleware/server"
 require "sidekiq_unique_jobs/sidekiq_unique_ext"
 require "sidekiq_unique_jobs/on_conflict"
+require "sidekiq_unique_jobs/changelog"
+require "sidekiq_unique_jobs/digests"
 
+require "sidekiq_unique_jobs/config"
 require "sidekiq_unique_jobs/sidekiq_unique_jobs"
+require "sidekiq_unique_jobs/update_version"
+require "sidekiq_unique_jobs/upgrade_locks"

data/lib/sidekiq_unique_jobs/batch_delete.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class BatchDelete provides batch deletion of digests
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class BatchDelete
+    #
+    # @return [Integer] the default batch size
+    BATCH_SIZE = 100
+
+    SUFFIXES = %w[
+      QUEUED
+      PRIMED
+      LOCKED
+      INFO
+    ].freeze
+
+    # includes "SidekiqUniqueJobs::Connection"
+    # @!parse include SidekiqUniqueJobs::Connection
+    include SidekiqUniqueJobs::Connection
+    # includes "SidekiqUniqueJobs::Logging"
+    # @!parse include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Logging
+
+    #
+    # @!attribute [r] digests
+    #   @return [Array<String>] a collection of digests to be deleted
+    attr_reader :digests
+    #
+    # @!attribute [r] conn
+    #   @return [Redis, RedisConnection, ConnectionPool] a redis connection
+    attr_reader :conn
+
+    #
+    # Executes a batch deletion of the provided digests
+    #
+    # @param [Array<String>] digests the digests to delete
+    # @param [Redis] conn the connection to use for deletion
+    #
+    # @return [void]
+    #
+    def self.call(digests, conn)
+      new(digests, conn).call
+    end
+
+    #
+    # Initialize a new batch delete instance
+    #
+    # @param [Array<String>] digests the digests to delete
+    # @param [Redis] conn the connection to use for deletion
+    #
+    def initialize(digests, conn)
+      @count   = 0
+      @digests = digests
+      @conn    = conn
+      @digests ||= []
+      @digests.compact!
+      redis_version # Avoid pipelined calling redis_version and getting a future.
+    end
+
+    #
+    # Executes a batch deletion of the provided digests
+    # @note Just wraps batch_delete to be able to provide no connection
+    #
+    #
+    def call
+      return log_info("Nothing to delete; exiting.") if digests.none?
+
+      log_info("Deleting batch with #{digests.size} digests")
+      return batch_delete(conn) if conn
+
+      redis { |rcon| batch_delete(rcon) }
+    end
+
+    private
+
+    #
+    # Does the actual batch deletion
+    #
+    #
+    # @return [Integer] the number of deleted digests
+    #
+    def batch_delete(conn)
+      digests.each_slice(BATCH_SIZE) do |chunk|
+        conn.pipelined do
+          chunk.each do |digest|
+            del_digest(conn, digest)
+            conn.zrem(SidekiqUniqueJobs::DIGESTS, digest)
+            @count += 1
+          end
+        end
+      end
+
+      @count
+    end
+
+    def del_digest(conn, digest)
+      removable_keys = keys_for_digest(digest)
+
+      if VersionCheck.satisfied?(redis_version, ">= 4.0.0")
+        conn.unlink(*removable_keys)
+      else
+        conn.del(*removable_keys)
+      end
+    end
+
+    def keys_for_digest(digest)
+      [digest, "#{digest}:RUN"].each_with_object([]) do |key, digest_keys|
+        digest_keys.concat([key])
+        digest_keys.concat(SUFFIXES.map { |suffix| "#{key}:#{suffix}" })
+      end
+    end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/changelog.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Changelogs provides access to the changelog entries
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class Changelog < Redis::SortedSet
+    def initialize
+      super(CHANGELOGS)
+    end
+
+    #
+    # Adds a new changelog entry
+    #
+    # @param [String] message a descriptive message about the entry
+    # @param [String] digest a unique digest
+    # @param [String] job_id a Sidekiq JID
+    # @param [String] script the name of the script adding the entry
+    #
+    # @return [void]
+    #
+    def add(message:, digest:, job_id:, script:)
+      message = dump_json(message: message, digest: digest, job_id: job_id, script: script)
+      redis { |conn| conn.zadd(key, now_f, message) }
+    end
+
+    #
+    # The change log entries
+    #
+    # @param [String] pattern the pattern to match
+    # @param [Integer] count the number of matches to return
+    #
+    # @return [Array<Hash>] an array of entries
+    #
+    def entries(pattern: "*", count: nil)
+      options = {}
+      options[:match] = pattern if pattern
+      options[:count] = count if count
+
+      redis do |conn|
+        conn.zscan_each(key, options).to_a.map { |entry| load_json(entry[0]) }
+      end
+    end
+
+    #
+    # Paginate the changelog entries
+    #
+    # @param [Integer] cursor the cursor for this iteration
+    # @param [String] pattern "*" the pattern to match
+    # @param [Integer] page_size 100 the number of matches to return
+    #
+    # @return [Array<Integer, Integer, Array<Hash>] the total size, next cursor and changelog entries
+    #
+    def page(cursor, pattern: "*", page_size: 100)
+      redis do |conn|
+        total_size, result = conn.multi do
+          conn.zcard(key)
+          conn.zscan(key, cursor, match: pattern, count: page_size)
+        end
+
+        [
+          total_size,
+          result[0], # next_cursor
+          result[1].map { |entry| load_json(entry[0]) }, # entries
+        ]
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/cli.rb

@@ -13,54 +13,45 @@ module SidekiqUniqueJobs
       "jobs #{@package_name} #{command.usage}"
     end
 
-    desc "keys PATTERN", "list all unique keys and their expiry time"
-    option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
-    def keys(pattern = "*")
-      keys = Util.keys(pattern, options[:count])
-      say "Found #{keys.size} keys matching '#{pattern}':"
-      print_in_columns(keys.sort) if keys.any?
+    desc "list PATTERN", "list all unique digests and their expiry time"
+    option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
+    def list(pattern = "*")
+      digests = SidekiqUniqueJobs::Digests.new.entries(pattern: pattern, count: options[:count])
+      say "Found #{digests.size} digests matching '#{pattern}':"
+      print_in_columns(digests.sort) if digests.any?
     end
 
-    desc "del PATTERN", "deletes unique keys from redis by pattern"
+    desc "del PATTERN", "deletes unique digests from redis by pattern"
     option :dry_run, aliases: :d, type: :boolean, desc: "set to false to perform deletion"
-    option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
+    option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
     def del(pattern)
       max_count = options[:count]
       if options[:dry_run]
-        keys = Util.keys(pattern, max_count)
-        say "Would delete #{keys.size} keys matching '#{pattern}'"
+        digests = SidekiqUniqueJobs::Digests.new.entries(pattern: pattern, count: max_count)
+        say "Would delete #{digests.size} digests matching '#{pattern}'"
       else
-        deleted_count = Util.del(pattern, max_count)
-        say "Deleted #{deleted_count} keys matching '#{pattern}'"
+        deleted_count = SidekiqUniqueJobs::Digests.new.del(pattern: pattern, count: max_count)
+        say "Deleted #{deleted_count} digests matching '#{pattern}'"
       end
     end
 
     desc "console", "drop into a console with easy access to helper methods"
     def console
-      say "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
-      say "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"
-      say "Use `del '*', 1000, false to delete the first 1000 keys matching '*'"
-      Object.include SidekiqUniqueJobs::Util
+      say "Use `list '*', 1000 to display the first 1000 unique digests matching '*'"
+      say "Use `del '*', 1000, true (default) to see how many digests would be deleted for the pattern '*'"
+      say "Use `del '*', 1000, false to delete the first 1000 digests matching '*'"
+
+      # Object.include SidekiqUniqueJobs::Api
       console_class.start
     end
 
     no_commands do
       def console_class
-        return irb if RUBY_PLATFORM == "JAVA"
-
-        pry
-      end
-
-      def irb
-        require "irb"
-        IRB
-      end
-
-      def pry
         require "pry"
         Pry
-      rescue LoadError, NameError
-        irb
+      rescue NameError, LoadError
+        require "irb"
+        IRB
       end
     end
   end

data/lib/sidekiq_unique_jobs/config.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # ThreadSafe config exists to be able to document the config class without errors
+  ThreadSafeConfig = Concurrent::MutableStruct.new("ThreadSafeConfig",
+                                                   :default_lock_timeout,
+                                                   :default_lock_ttl,
+                                                   :enabled,
+                                                   :unique_prefix,
+                                                   :logger,
+                                                   :locks,
+                                                   :strategies,
+                                                   :debug_lua,
+                                                   :max_history,
+                                                   :reaper,
+                                                   :reaper_count,
+                                                   :reaper_interval,
+                                                   :reaper_timeout,
+                                                   :lock_info,
+                                                   :raise_on_config_error,
+                                                   :current_redis_version)
+
+  #
+  # Shared class for dealing with gem configuration
+  #
+  # @author Mauro Berlanda <mauro.berlanda@gmail.com>
+  class Config < ThreadSafeConfig
+    LOCKS_WHILE_ENQUEUED = {
+      until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
+      while_enqueued: SidekiqUniqueJobs::Lock::UntilExecuting,
+    }.freeze
+
+    LOCKS_FROM_PUSH_TO_PROCESSED = {
+      until_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_performed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_processed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
+      until_successfully_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    }.freeze
+
+    LOCKS_WITHOUT_UNLOCK = {
+      until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
+    }.freeze
+
+    LOCKS_WHEN_BUSY = {
+      around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_busy: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_working: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
+    }.freeze
+
+    LOCKS =
+      LOCKS_WHEN_BUSY.dup
+                     .merge(LOCKS_WHILE_ENQUEUED.dup)
+                     .merge(LOCKS_WITHOUT_UNLOCK.dup)
+                     .merge(LOCKS_FROM_PUSH_TO_PROCESSED.dup)
+                     .freeze
+
+    STRATEGIES = {
+      log: SidekiqUniqueJobs::OnConflict::Log,
+      raise: SidekiqUniqueJobs::OnConflict::Raise,
+      reject: SidekiqUniqueJobs::OnConflict::Reject,
+      replace: SidekiqUniqueJobs::OnConflict::Replace,
+      reschedule: SidekiqUniqueJobs::OnConflict::Reschedule,
+    }.freeze
+
+    PREFIX                = "uniquejobs"
+    LOCK_TIMEOUT          = 0
+    LOCK_TTL              = nil
+    ENABLED               = true
+    DEBUG_LUA             = false
+    MAX_HISTORY           = 1_000
+    REAPER                = :ruby # The type of cleanup to run. Possible values are [:ruby, :lua]
+    REAPER_COUNT          = 1_000
+    REAPER_INTERVAL       = 600 # Every 10 minutes
+    REAPER_TIMEOUT        = 10 # 10 seconds
+    USE_LOCK_INFO         = false
+    RAISE_ON_CONFIG_ERROR = false
+    REDIS_VERSION         = "0.0.0"
+
+    #
+    # Returns a default configuration
+    #
+    # @example
+    #   SidekiqUniqueJobs::Config.default => <concurrent/mutable_struct/thread_safe_config SidekiqUniqueJobs::Config {
+    #   default_lock_timeout: 0,
+    #   default_lock_ttl: nil,
+    #   enabled: true,
+    #   unique_prefix: "uniquejobs",
+    #   logger: #<Sidekiq::Logger:0x00007f81e096b0e0 @level=1 ...>,
+    #   locks: {
+    #     around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_busy: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_working: SidekiqUniqueJobs::Lock::WhileExecuting,
+    #     while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
+    #     until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
+    #     while_enqueued: SidekiqUniqueJobs::Lock::UntilExecuting,
+    #     until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
+    #     until_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_performed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_processed: SidekiqUniqueJobs::Lock::UntilExecuted,
+    #     until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
+    #     until_successfully_completed: SidekiqUniqueJobs::Lock::UntilExecuted
+    #   },
+    #   strategies: {
+    #     log: SidekiqUniqueJobs::OnConflict::Log,
+    #     raise: SidekiqUniqueJobs::OnConflict::Raise,
+    #     reject: SidekiqUniqueJobs::OnConflict::Reject,
+    #     replace: SidekiqUniqueJobs::OnConflict::Replace,
+    #     reschedule: SidekiqUniqueJobs::OnConflict::Reschedule
+    #   },
+    #   debug_lua: false,
+    #   max_history: 1000,
+    #   reaper:: ruby,
+    #   reaper_count: 1000,
+    #   lock_info: false,
+    #   raise_on_config_error: false,
+    #   }>
+    #
+    #
+    # @return [SidekiqUniqueJobs::Config] a default configuration
+    #
+    def self.default # rubocop:disable Metrics/MethodLength
+      new(
+        LOCK_TIMEOUT,
+        LOCK_TTL,
+        ENABLED,
+        PREFIX,
+        Sidekiq.logger,
+        LOCKS,
+        STRATEGIES,
+        DEBUG_LUA,
+        MAX_HISTORY,
+        REAPER,
+        REAPER_COUNT,
+        REAPER_INTERVAL,
+        REAPER_TIMEOUT,
+        USE_LOCK_INFO,
+        RAISE_ON_CONFIG_ERROR,
+        REDIS_VERSION,
+      )
+    end
+
+    #
+    # Adds a lock type to the configuration. It will raise if the lock exists already
+    #
+    # @example Add a custom lock
+    #   add_lock(:my_lock, CustomLocks::MyLock)
+    #
+    # @raise DuplicateLock when the name already exists
+    #
+    # @param [String, Symbol] name the name of the lock
+    # @param [Class] klass the class describing the lock
+    #
+    # @return [void]
+    #
+    def add_lock(name, klass)
+      lock_sym = name.to_sym
+      raise DuplicateLock, ":#{name} already defined, please use another name" if locks.key?(lock_sym)
+
+      new_locks = locks.dup.merge(lock_sym => klass).freeze
+      self.locks = new_locks
+    end
+
+    #
+    # Adds an on_conflict strategy to the configuration.
+    #
+    # @example Add a custom strategy
+    #   add_lock(:my_strategy, CustomStrategies::MyStrategy)
+    #
+    # @raise [DuplicateStrategy] when the name already exists
+    #
+    # @param [String] name the name of the custom strategy
+    # @param [Class] klass the class describing the strategy
+    #
+    def add_strategy(name, klass)
+      strategy_sym = name.to_sym
+      raise DuplicateStrategy, ":#{name} already defined, please use another name" if strategies.key?(strategy_sym)
+
+      new_strategies = strategies.dup.merge(strategy_sym => klass).freeze
+      self.strategies = new_strategies
+    end
+
+    def redis_version
+      self.current_redis_version = SidekiqUniqueJobs.fetch_redis_version if current_redis_version == REDIS_VERSION
+      current_redis_version
+    end
+  end
+end