sidekiq-unique-jobs 4.0.18 → 7.1.28

data/bin/uniquejobs

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "sidekiq_unique_jobs"
+
+SidekiqUniqueJobs::Cli.start(ARGV)

data/lib/sidekiq-unique-jobs.rb

@@ -1,79 +1,3 @@
-require 'yaml' if RUBY_VERSION.include?('2.0.0') # rubocop:disable FileName
-require 'forwardable'
-require 'sidekiq_unique_jobs/version'
-require 'sidekiq_unique_jobs/constants'
-require 'sidekiq_unique_jobs/util'
-require 'sidekiq_unique_jobs/cli'
-require 'sidekiq_unique_jobs/core_ext'
-require 'sidekiq_unique_jobs/timeout_calculator'
-require 'sidekiq_unique_jobs/options_with_fallback'
-require 'sidekiq_unique_jobs/scripts'
-require 'sidekiq_unique_jobs/unique_args'
-require 'sidekiq_unique_jobs/unlockable'
-require 'sidekiq_unique_jobs/lock'
-require 'sidekiq_unique_jobs/middleware'
-require 'sidekiq_unique_jobs/config'
-require 'sidekiq_unique_jobs/sidekiq_unique_ext'
+# frozen_string_literal: true
 
-require 'ostruct'
-
-module SidekiqUniqueJobs
-  module_function
-
-  def config
-    @config ||= Config.new(
-      unique_prefix: 'uniquejobs',
-      default_queue_lock_expiration: 30 * 60,
-      default_run_lock_expiration: 60,
-      default_lock: :while_executing,
-      redis_test_mode: :redis # :mock
-    )
-  end
-
-  def default_lock
-    config.default_lock
-  end
-
-  def configure(options = {})
-    if block_given?
-      yield config
-    else
-      options.each do |key, val|
-        config.send("#{key}=", val)
-      end
-    end
-  end
-
-  def namespace
-    @namespace ||= connection { |c| c.respond_to?(:namespace) ? c.namespace : nil }
-  end
-
-  # Attempt to constantize a string worker_class argument, always
-  # failing back to the original argument.
-  def worker_class_constantize(worker_class)
-    return worker_class unless worker_class.is_a?(String)
-    worker_class.constantize
-  rescue NameError
-    worker_class
-  end
-
-  def mocked?
-    config.redis_test_mode == :mock
-  end
-
-  def redis_version
-    @redis_version ||= connection { |c| c.info('server'.freeze)['redis_version'.freeze] }
-  end
-
-  def connection(redis_pool = nil)
-    if redis_pool
-      redis_pool.with { |conn| yield conn }
-    else
-      Sidekiq.redis { |conn| yield conn }
-    end
-  end
-
-  def synchronize(item, redis_pool)
-    Lock::WhileExecuting.synchronize(item, redis_pool) { yield }
-  end
-end
+require "sidekiq_unique_jobs"

data/lib/sidekiq_unique_jobs/batch_delete.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class BatchDelete provides batch deletion of digests
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class BatchDelete
+    #
+    # @return [Integer] the default batch size
+    BATCH_SIZE = 100
+
+    #
+    # @return [Array<String>] Supported key suffixes
+    SUFFIXES = %w[
+      QUEUED
+      PRIMED
+      LOCKED
+      INFO
+    ].freeze
+
+    # includes "SidekiqUniqueJobs::Connection"
+    # @!parse include SidekiqUniqueJobs::Connection
+    include SidekiqUniqueJobs::Connection
+    # includes "SidekiqUniqueJobs::Logging"
+    # @!parse include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Logging
+
+    #
+    # @!attribute [r] digests
+    #   @return [Array<String>] a collection of digests to be deleted
+    attr_reader :digests
+    #
+    # @!attribute [r] conn
+    #   @return [Redis, RedisConnection, ConnectionPool] a redis connection
+    attr_reader :conn
+
+    #
+    # Executes a batch deletion of the provided digests
+    #
+    # @param [Array<String>] digests the digests to delete
+    # @param [Redis] conn the connection to use for deletion
+    #
+    # @return [void]
+    #
+    def self.call(digests, conn = nil)
+      new(digests, conn).call
+    end
+
+    #
+    # Initialize a new batch delete instance
+    #
+    # @param [Array<String>] digests the digests to delete
+    # @param [Redis] conn the connection to use for deletion
+    #
+    def initialize(digests, conn)
+      @count   = 0
+      @digests = digests
+      @conn    = conn
+      @digests ||= []
+      @digests.compact!
+      redis_version # Avoid pipelined calling redis_version and getting a future.
+    end
+
+    #
+    # Executes a batch deletion of the provided digests
+    # @note Just wraps batch_delete to be able to provide no connection
+    #
+    #
+    def call
+      return log_info("Nothing to delete; exiting.") if digests.none?
+
+      log_info("Deleting batch with #{digests.size} digests")
+      return batch_delete(conn) if conn
+
+      redis { |rcon| batch_delete(rcon) }
+    end
+
+    private
+
+    #
+    # Does the actual batch deletion
+    #
+    #
+    # @return [Integer] the number of deleted digests
+    #
+    def batch_delete(conn)
+      digests.each_slice(BATCH_SIZE) do |chunk|
+        conn.pipelined do |pipeline|
+          chunk.each do |digest|
+            del_digest(pipeline, digest)
+            pipeline.zrem(SidekiqUniqueJobs::DIGESTS, digest)
+            pipeline.zrem(SidekiqUniqueJobs::EXPIRING_DIGESTS, digest)
+            @count += 1
+          end
+        end
+      end
+
+      @count
+    end
+
+    def del_digest(pipeline, digest)
+      removable_keys = keys_for_digest(digest)
+
+      if VersionCheck.satisfied?(redis_version, ">= 4.0.0")
+        pipeline.unlink(*removable_keys)
+      else
+        pipeline.del(*removable_keys)
+      end
+    end
+
+    def keys_for_digest(digest)
+      [digest, "#{digest}:RUN"].each_with_object([]) do |key, digest_keys|
+        digest_keys.concat([key])
+        digest_keys.concat(SUFFIXES.map { |suffix| "#{key}:#{suffix}" })
+      end
+    end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/changelog.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Changelogs provides access to the changelog entries
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class Changelog < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
+    DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
+    SCAN_PATTERN  = "*"
+
+    def initialize
+      super(CHANGELOGS)
+    end
+
+    #
+    # Adds a new changelog entry
+    #
+    # @param [String] message a descriptive message about the entry
+    # @param [String] digest a unique digest
+    # @param [String] job_id a Sidekiq JID
+    # @param [String] script the name of the script adding the entry
+    #
+    # @return [void]
+    #
+    def add(message:, digest:, job_id:, script:)
+      message = dump_json(message: message, digest: digest, job_id: job_id, script: script)
+      redis { |conn| conn.zadd(key, now_f, message) }
+    end
+
+    #
+    # The change log entries
+    #
+    # @param [String] pattern the pattern to match
+    # @param [Integer] count the number of matches to return
+    #
+    # @return [Array<Hash>] an array of entries
+    #
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
+      options = {}
+      options[:match] = pattern
+      options[:count] = count
+
+      redis do |conn|
+        conn.zscan_each(key, **options).to_a.map { |entry| load_json(entry[0]) }
+      end
+    end
+
+    #
+    # Paginate the changelog entries
+    #
+    # @param [Integer] cursor the cursor for this iteration
+    # @param [String] pattern "*" the pattern to match
+    # @param [Integer] page_size 100 the number of matches to return
+    #
+    # @return [Array<Integer, Integer, Array<Hash>] the total size, next cursor and changelog entries
+    #
+    def page(cursor: 0, pattern: "*", page_size: 100)
+      redis do |conn|
+        total_size, result = conn.multi do |pipeline|
+          pipeline.zcard(key)
+          pipeline.zscan(key, cursor, match: pattern, count: page_size)
+        end
+
+        [
+          total_size.to_i,
+          result[0].to_i, # next_cursor
+          result[1].map { |entry| load_json(entry[0]) }, # entries
+        ]
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/cli.rb

@@ -1,37 +1,95 @@
-require 'thor'
+# frozen_string_literal: true
+
+require "thor"
 
 module SidekiqUniqueJobs
+  #
+  # Command line interface for unique jobs
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
   class Cli < Thor
-    desc 'keys PATTERN', 'list all unique keys and their expiry time'
-    option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
-    def keys(pattern)
-      Util.keys(pattern, options[:count])
+    # :nodoc:
+    # rubocop:disable Style/OptionalBooleanParameter
+    def self.banner(command, _namespace = nil, _subcommand = false) # rubocop:disable Style/OptionalBooleanParameter
+      "jobs #{@package_name} #{command.usage}" # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
+    end
+    # rubocop:enable Style/OptionalBooleanParameter
+
+    desc "list PATTERN", "list all unique digests and their expiry time"
+    option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
+    # :nodoc:
+    def list(pattern = "*")
+      max_count = options[:count]
+      say "Searching for regular digests"
+      list_entries(digests.entries(pattern: pattern, count: max_count), pattern)
+      say "Searching for expiring digests"
+      list_entries(expiring_digests.entries(pattern: pattern, count: max_count), pattern)
     end
 
-    desc 'del PATTERN', 'deletes unique keys from redis by pattern'
-    option :dry_run, aliases: :d, type: :boolean, desc: 'set to false to perform deletion'
-    option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
+    desc "del PATTERN", "deletes unique digests from redis by pattern"
+    option :dry_run, aliases: :d, type: :boolean, desc: "set to false to perform deletion"
+    option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
+    # :nodoc:
     def del(pattern)
-      Util.del(pattern, options[:count], options[:dry_run])
+      max_count = options[:count]
+      if options[:dry_run]
+        count_entries_for_del(max_count, pattern)
+      else
+        del_entries(max_count, pattern)
+      end
     end
 
-    desc 'console', 'drop into a console with easy access to helper methods'
+    desc "console", "drop into a console with easy access to helper methods"
+    # :nodoc:
     def console
-      puts "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
-      puts "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"
-      puts "Use `del '*', 1000, false to delete the first 1000 keys matching '*'"
-      Object.include SidekiqUniqueJobs::Util
+      say "Use `list '*', 1000 to display the first 1000 unique digests matching '*'"
+      say "Use `del '*', 1000, true (default) to see how many digests would be deleted for the pattern '*'"
+      say "Use `del '*', 1000, false to delete the first 1000 digests matching '*'"
+
+      # Object.include SidekiqUniqueJobs::Api
       console_class.start
     end
 
-    private
+    no_commands do # rubocop:disable Metrics/BlockLength
+      # :nodoc:
+      def digests
+        @digests ||= SidekiqUniqueJobs::Digests.new
+      end
+
+      # :nodoc:
+      def expiring_digests
+        @expiring_digests ||= SidekiqUniqueJobs::ExpiringDigests.new
+      end
+
+      # :nodoc:
+      def console_class
+        require "pry"
+        Pry
+      rescue NameError, LoadError
+        require "irb"
+        IRB
+      end
+
+      # :nodoc:
+      def list_entries(entries, pattern)
+        say "Found #{entries.size} digests matching '#{pattern}':"
+        print_in_columns(entries.sort) if entries.any?
+      end
+
+      # :nodoc:
+      def count_entries_for_del(max_count, pattern)
+        count = digests.entries(pattern: pattern, count: max_count).size +
+                expiring_digests.entries(pattern: pattern, count: max_count).size
+        say "Would delete #{count} digests matching '#{pattern}'"
+      end
 
-    def console_class
-      require 'pry'
-      Pry
-    rescue LoadError
-      require 'irb'
-      IRB
+      # :nodoc:
+      def del_entries(max_count, pattern)
+        deleted_count = digests.delete_by_pattern(pattern, count: max_count).to_i +
+                        expiring_digests.delete_by_pattern(pattern, count: max_count).to_i
+        say "Deleted #{deleted_count} digests matching '#{pattern}'"
+      end
     end
   end
 end