sidekiq-unique-jobs 6.0.23 → 7.1.12

data/lib/sidekiq_unique_jobs/client/middleware.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require "sidekiq_unique_jobs/server/middleware"
-
-module SidekiqUniqueJobs
-  module Client
-    # The unique sidekiq middleware for the client push
-    #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
-    class Middleware
-      include SidekiqUniqueJobs::Logging
-      include OptionsWithFallback
-
-      # Calls this client middleware
-      #   Used from Sidekiq.process_single
-      # @param [String] worker_class name of the sidekiq worker class
-      # @param [Hash] item a sidekiq job hash
-      # @param [String] queue name of the queue
-      # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
-      # @yield when uniqueness is disable or lock successful
-      def call(worker_class, item, queue, redis_pool = nil)
-        @worker_class = worker_class
-        @item         = item
-        @queue        = queue
-        @redis_pool   = redis_pool
-
-        yield if success?
-      end
-
-      private
-
-      # The sidekiq job hash
-      # @return [Hash] the Sidekiq job hash
-      attr_reader :item
-
-      def success?
-        unique_disabled? || locked?
-      end
-
-      def locked?
-        SidekiqUniqueJobs::Job.add_uniqueness(item)
-        SidekiqUniqueJobs.with_context(logging_context(self.class, item)) do
-          locked = lock.lock
-          warn_about_duplicate unless locked
-          locked
-        end
-      end
-
-      def warn_about_duplicate
-        return unless log_duplicate_payload?
-
-        log_warn "payload is not unique #{item}"
-      end
-    end
-  end
-end

data/lib/sidekiq_unique_jobs/scripts.rb

@@ -1,118 +0,0 @@
-# frozen_string_literal: true
-
-require "pathname"
-require "digest/sha1"
-require "concurrent/map"
-
-module SidekiqUniqueJobs
-  # Interface to dealing with .lua files
-  #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  module Scripts
-    LUA_PATHNAME ||= Pathname.new(__FILE__).dirname.join("../../redis").freeze
-    SCRIPT_SHAS ||= Concurrent::Map.new
-
-    include SidekiqUniqueJobs::Connection
-
-    module_function
-
-    #
-    # Call a lua script with the provided file_name
-    #
-    # @note this method is recursive if we need to load a lua script
-    #   that wasn't previously loaded.
-    #
-    # @param [Symbol] file_name the name of the lua script
-    # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
-    # @param [Hash] options arguments to pass to the script file
-    # @option options [Array] :keys the array of keys to pass to the script
-    # @option options [Array] :argv the array of arguments to pass to the script
-    #
-    # @return value from script
-    #
-    def call(file_name, redis_pool, options = {})
-      execute_script(file_name, redis_pool, options)
-    rescue Redis::CommandError => ex
-      handle_error(ex, file_name) do
-        call(file_name, redis_pool, options)
-      end
-    end
-
-    #
-    # Execute the script file
-    #
-    # @param [Symbol] file_name the name of the lua script
-    # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
-    # @param [Hash] options arguments to pass to the script file
-    # @option options [Array] :keys the array of keys to pass to the script
-    # @option options [Array] :argv the array of arguments to pass to the script
-    #
-    # @return value from script (evalsha)
-    #
-    def execute_script(file_name, redis_pool, options = {})
-      redis(redis_pool) do |conn|
-        sha = script_sha(conn, file_name)
-        conn.evalsha(sha, options)
-      end
-    end
-
-    #
-    # Return sha of already loaded lua script or load it and return the sha
-    #
-    # @param [Sidekiq::RedisConnection] conn the redis connection
-    # @param [Symbol] file_name the name of the lua script
-    # @return [String] sha of the script file
-    #
-    # @return [String] the sha of the script
-    #
-    def script_sha(conn, file_name)
-      if (sha = SCRIPT_SHAS.get(file_name))
-        return sha
-      end
-
-      sha = conn.script(:load, script_source(file_name))
-      SCRIPT_SHAS.put(file_name, sha)
-      sha
-    end
-
-    #
-    # Handle errors to allow retrying errors that need retrying
-    #
-    # @param [Redis::CommandError] ex exception to handle
-    # @param [Symbol] file_name the name of the lua script
-    #
-    # @return [void]
-    #
-    # @yieldreturn [void] yields back to the caller when NOSCRIPT is raised
-    def handle_error(ex, file_name)
-      if ex.message == "NOSCRIPT No matching script. Please use EVAL."
-        SCRIPT_SHAS.delete(file_name)
-        return yield if block_given?
-      end
-
-      raise ScriptError, file_name: file_name, source_exception: ex
-    end
-
-    #
-    # Reads the lua file from disk
-    #
-    # @param [Symbol] file_name the name of the lua script
-    #
-    # @return [String] the content of the lua file
-    #
-    def script_source(file_name)
-      script_path(file_name).read
-    end
-
-    #
-    # Construct a Pathname to a lua script
-    #
-    # @param [Symbol] file_name the name of the lua script
-    #
-    # @return [Pathname] the full path to the gems lua script
-    #
-    def script_path(file_name)
-      LUA_PATHNAME.join("#{file_name}.lua")
-    end
-  end
-end

data/lib/sidekiq_unique_jobs/server/middleware.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module SidekiqUniqueJobs
-  module Server
-    # The unique sidekiq middleware for the server processor
-    #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
-    class Middleware
-      include Logging
-      include OptionsWithFallback
-
-      #
-      #
-      # Runs the server middleware (used from Sidekiq::Processor#process)
-      #
-      # @param [Sidekiq::Worker] worker_class
-      # @param [Hash] item a sidekiq job hash
-      # @param [String] queue name of the queue
-      #
-      # @see https://github.com/mperham/sidekiq/wiki/Job-Format
-      # @see https://github.com/mperham/sidekiq/wiki/Middleware
-      #
-      # @yield when uniqueness is disabled
-      # @yield when the lock is acquired
-      def call(worker_class, item, queue)
-        @worker_class = worker_class
-        @item         = item
-        @queue        = queue
-        return yield if unique_disabled?
-
-        SidekiqUniqueJobs::Job.add_uniqueness(item)
-        SidekiqUniqueJobs.with_context(logging_context(self.class, item)) do
-          lock.execute do
-            yield
-          end
-        end
-      end
-
-      private
-
-      # The sidekiq job hash
-      # @return [Hash] the Sidekiq job hash
-      attr_reader :item
-    end
-  end
-end

data/lib/sidekiq_unique_jobs/timeout/calculator.rb

@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-module SidekiqUniqueJobs
-  module Timeout
-    # Calculates timeout and expiration
-    #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
-    class Calculator
-      include SidekiqUniqueJobs::SidekiqWorkerMethods
-
-      # @attr [Hash] item the Sidekiq job hash
-      attr_reader :item
-
-      # @param [Hash] item the Sidekiq job hash
-      # @option item [Integer, nil] :lock_expiration the configured lock expiration
-      # @option item [Integer, nil] :lock_timeout the configured lock timeout
-      # @option item [String] :class the class of the sidekiq worker
-      # @option item [Float] :at the unix time the job is scheduled at
-      def initialize(item)
-        @item         = item
-        @worker_class = item[CLASS_KEY]
-      end
-
-      # The time until a job is scheduled
-      # @return [Integer] the number of seconds until job is scheduled
-      def time_until_scheduled
-        return 0 unless scheduled_at
-
-        scheduled_at.to_i - Time.now.utc.to_i
-      end
-
-      # The time a job is scheduled
-      # @return [Float] the exact unix time the job is scheduled at
-      def scheduled_at
-        @scheduled_at ||= item[AT_KEY]
-      end
-
-      # The configured lock_expiration
-      def lock_expiration
-        @lock_expiration ||= begin
-          expiration = item[LOCK_EXPIRATION_KEY]
-          expiration ||= worker_options[LOCK_EXPIRATION_KEY]
-          expiration && expiration.to_i + time_until_scheduled
-        end
-      end
-
-      # The configured lock_timeout
-      def lock_timeout
-        @lock_timeout = begin
-          timeout = default_worker_options[LOCK_TIMEOUT_KEY]
-          timeout = default_lock_timeout if default_lock_timeout
-          timeout = worker_options[LOCK_TIMEOUT_KEY] if worker_options.key?(LOCK_TIMEOUT_KEY)
-          timeout
-        end
-      end
-
-      # The default lock_timeout of this gem
-      def default_lock_timeout
-        SidekiqUniqueJobs.config.default_lock_timeout
-      end
-    end
-  end
-end

data/lib/sidekiq_unique_jobs/timeout.rb

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-module SidekiqUniqueJobs
-  module Timeout
-  end
-end
-
-require "sidekiq_unique_jobs/timeout/calculator"

data/lib/sidekiq_unique_jobs/unique_args.rb

@@ -1,150 +0,0 @@
-# frozen_string_literal: true
-
-require "digest"
-require "openssl"
-require "sidekiq_unique_jobs/normalizer"
-
-module SidekiqUniqueJobs
-  # Handles uniqueness of sidekiq arguments
-  #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class UniqueArgs
-    include SidekiqUniqueJobs::Logging
-    include SidekiqUniqueJobs::SidekiqWorkerMethods
-
-    # Convenience method for returning a digest
-    # @param [Hash] item a Sidekiq job hash
-    # @return [String] a unique digest
-    def self.digest(item)
-      new(item).unique_digest
-    end
-
-    # The sidekiq job hash
-    # @return [Hash] the Sidekiq job hash
-    attr_reader :item
-
-    # @param [Hash] item a Sidekiq job hash
-    def initialize(item)
-      @item         = item
-      @worker_class = item[CLASS_KEY]
-
-      add_uniqueness_to_item
-    end
-
-    # Appends the keys unique_prefix, unique_args and {#unique_digest} to the sidekiq job hash {#item}
-    # @return [void]
-    def add_uniqueness_to_item
-      item[UNIQUE_PREFIX_KEY] ||= unique_prefix
-      item[UNIQUE_ARGS_KEY]     = unique_args(item[ARGS_KEY])
-      item[UNIQUE_DIGEST_KEY]   = unique_digest
-    end
-
-    # Memoized unique_digest
-    # @return [String] a unique digest
-    def unique_digest
-      @unique_digest ||= create_digest
-    end
-
-    # Creates a namespaced unique digest based on the {#digestable_hash} and the {#unique_prefix}
-    # @return [String] a unique digest
-    def create_digest
-      digest = OpenSSL::Digest::MD5.hexdigest(Sidekiq.dump_json(digestable_hash))
-      "#{unique_prefix}:#{digest}"
-    end
-
-    # A prefix to use as namespace for the {#unique_digest}
-    # @return [String] a unique digest
-    def unique_prefix
-      worker_options[UNIQUE_PREFIX_KEY] || SidekiqUniqueJobs.config.unique_prefix
-    end
-
-    # Filter a hash to use for digest
-    # @return [Hash] to use for digest
-    def digestable_hash
-      @item.slice(CLASS_KEY, QUEUE_KEY, UNIQUE_ARGS_KEY).tap do |hash|
-        hash.delete(QUEUE_KEY) if unique_across_queues?
-        hash.delete(CLASS_KEY) if unique_across_workers?
-      end
-    end
-
-    # The unique arguments to use for creating a lock
-    # @return [Array] the arguments filters by the {#filtered_args} method if {#unique_args_enabled?}
-    def unique_args(args)
-      return filtered_args(args) if unique_args_enabled?
-
-      args
-    end
-
-    # Checks if we should disregard the queue when creating the unique digest
-    # @return [true, false]
-    def unique_across_queues?
-      item[UNIQUE_ACROSS_QUEUES_KEY] || worker_options[UNIQUE_ACROSS_QUEUES_KEY] ||
-        item[UNIQUE_ON_ALL_QUEUES_KEY] || worker_options[UNIQUE_ON_ALL_QUEUES_KEY] # TODO: Remove in v 6.1
-    end
-
-    # Checks if we should disregard the worker when creating the unique digest
-    # @return [true, false]
-    def unique_across_workers?
-      item[UNIQUE_ACROSS_WORKERS_KEY] || worker_options[UNIQUE_ACROSS_WORKERS_KEY]
-    end
-
-    # Checks if the worker class has been enabled for unique_args?
-    # @return [true, false]
-    def unique_args_enabled?
-      unique_args_method # && !unique_args_method.is_a?(Boolean)
-    end
-
-    # Filters unique arguments by proc or symbol
-    # @param [Array] args the arguments passed to the sidekiq worker
-    # @return [Array] {#filter_by_proc} when {#unique_args_method} is a Proc
-    # @return [Array] {#filter_by_symbol} when {#unique_args_method} is a Symbol
-    # @return [Array] args unfiltered when neither of the above
-    def filtered_args(args)
-      return args if args.empty?
-
-      json_args = Normalizer.jsonify(args)
-
-      case unique_args_method
-      when Proc
-        filter_by_proc(json_args)
-      when Symbol
-        filter_by_symbol(json_args)
-      else
-        log_debug("#{__method__} arguments not filtered (using all arguments for uniqueness)")
-        json_args
-      end
-    end
-
-    # Filters unique arguments by proc configured in the sidekiq worker
-    # @param [Array] args the arguments passed to the sidekiq worker
-    # @return [Array] with the filtered arguments
-    def filter_by_proc(args)
-      unique_args_method.call(args)
-    end
-
-    # Filters unique arguments by method configured in the sidekiq worker
-    # @param [Array] args the arguments passed to the sidekiq worker
-    # @return [Array] unfiltered unless {#worker_method_defined?}
-    # @return [Array] with the filtered arguments
-    def filter_by_symbol(args)
-      return args unless worker_method_defined?(unique_args_method)
-
-      worker_class.send(unique_args_method, args)
-    rescue ArgumentError => ex
-      log_fatal(ex)
-      args
-    end
-
-    # The method to use for filtering unique arguments
-    def unique_args_method
-      @unique_args_method ||= worker_options[UNIQUE_ARGS_KEY]
-      @unique_args_method ||= :unique_args if worker_method_defined?(:unique_args)
-      @unique_args_method ||= default_unique_args_method
-    end
-
-    # The global worker options defined in Sidekiq directly
-    def default_unique_args_method
-      Sidekiq.default_worker_options.stringify_keys[UNIQUE_ARGS_KEY]
-    end
-  end
-end

data/lib/sidekiq_unique_jobs/util.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-module SidekiqUniqueJobs
-  # Utility module to help manage unique keys in redis.
-  # Useful for deleting keys that for whatever reason wasn't deleted
-  #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  module Util
-    DEFAULT_COUNT     = 1_000
-    SCAN_PATTERN      = "*"
-
-    include SidekiqUniqueJobs::Logging
-    include SidekiqUniqueJobs::Connection
-    extend self
-
-    # Find unique keys in redis
-    #
-    # @param [String] pattern a pattern to scan for in redis
-    # @param [Integer] count the maximum number of keys to delete
-    # @return [Array<String>] an array with active unique keys
-    def keys(pattern = SCAN_PATTERN, count = DEFAULT_COUNT)
-      return redis(&:keys) if pattern.nil?
-
-      redis { |conn| conn.scan_each(match: prefix(pattern), count: count).to_a }
-    end
-
-    # Find unique keys with ttl
-    # @param [String] pattern a pattern to scan for in redis
-    # @param [Integer] count the maximum number of keys to delete
-    # @return [Hash<String, Integer>] a hash with active unique keys and corresponding ttl
-    def keys_with_ttl(pattern = SCAN_PATTERN, count = DEFAULT_COUNT)
-      hash = {}
-      redis do |conn|
-        conn.scan_each(match: prefix(pattern), count: count).each do |key|
-          hash[key] = conn.ttl(key)
-        end
-      end
-      hash
-    end
-
-    # Deletes unique keys from redis
-    #
-    # @param [String] pattern a pattern to scan for in redis
-    # @param [Integer] count the maximum number of keys to delete
-    # @return [Integer] the number of keys deleted
-    def del(pattern = SCAN_PATTERN, count = 0)
-      raise ArgumentError, "Please provide a number of keys to delete greater than zero" if count.zero?
-
-      pattern = suffix(pattern)
-
-      log_debug { "Deleting keys by: #{pattern}" }
-      keys, time = timed { keys(pattern, count) }
-      key_size   = keys.size
-      log_debug { "#{key_size} keys found in #{time} sec." }
-      _, time = timed { batch_delete(keys) }
-      log_debug { "Deleted #{key_size} keys in #{time} sec." }
-
-      key_size
-    end
-
-    private
-
-    def batch_delete(keys)
-      redis do |conn|
-        keys.each_slice(500) do |chunk|
-          conn.pipelined do
-            chunk.each do |key|
-              conn.del key
-            end
-          end
-        end
-      end
-    end
-
-    def timed
-      start   = current_time
-      result  = yield
-      elapsed = (current_time - start).round(2)
-      [result, elapsed]
-    end
-
-    def current_time
-      Time.now
-    end
-
-    def prefix(key)
-      return key if unique_prefix.nil?
-      return key if key.start_with?("#{unique_prefix}:")
-
-      "#{unique_prefix}:#{key}"
-    end
-
-    def suffix(key)
-      return "#{key}*" unless key.end_with?(":*")
-
-      key
-    end
-
-    def unique_prefix
-      SidekiqUniqueJobs.config.unique_prefix
-    end
-  end
-end

data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb

@@ -1,28 +0,0 @@
-<header class="row">
-  <div class="col-sm-5">
-    <h3><%= t('Unique Digest') %> - <a class="btn btn-default btn-xs" href="<%= root_path %>unique_digests"><%= t('GoBack') %></a></h3>
-  </div>
-</header>
-
-<% if @unique_keys.size.positive? %>
-  <div class="table_container">
-    <table class="table table-striped table-bordered table-hover">
-      <thead>
-        <tr>
-          <th>
-            <%= t('Keys') %>
-            <span class="small text-muted">for (<%= @digest %>)</span>
-          </th>
-        </tr>
-      </thead>
-        <% @unique_keys.each do |key| %>
-          <tr><td colspan="2"><%= key %></td></tr>
-        <% end %>
-    </table>
-  </div>
-  <form action="<%= root_path %>unique_digests/<%= @digest %>/delete" method="get">
-    <%= csrf_tag %>
-    <a class="btn btn-default btn-xs" href="<%= root_path %>unique_digests"><%= t('GoBack') %></a>
-    <input class="btn btn-danger btn-xs flip" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
-  </form>
-<% end %>

data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb

@@ -1,46 +0,0 @@
-<header class="row">
-  <div class="col-sm-5">
-    <h3><%= t('Unique Digests') %></h3>
-  </div>
-  <form action="<%= root_path %>unique_digests" class="form form-inline" method="get">
-    <%= csrf_tag %>
-    <input name="filter" class="form-control" type="text" value="<%= @filter %>" />
-    <button class="btn btn-default" type="submit"><%= t('Filter') %></button>
-  </form>
-  <% if @unique_digests.size > 0 && @total_size > @count.to_i %>
-    <div class="col-sm-4">
-      <%= erb unique_template(:_paging), locals: { url: "#{root_path}unique_digests" } %>
-    </div>
-  <% end %>
-</header>
-
-<% if @unique_digests.size.positive? %>
-  <div class="table_container">
-    <table class="table table-striped table-bordered table-hover">
-      <thead>
-        <tr>
-          <th><%= t('Delete') %></th>
-          <th><%= t('Digest') %></th>
-        </tr>
-      </thead>
-      <% @unique_digests.each do |digest| %>
-        <tr>
-          <td>
-            <form action="<%= root_path %>unique_digests/<%= digest %>/delete" method="get">
-              <%= csrf_tag %>
-              <input name="digest" value="<%= h digest %>" type="hidden" />
-              <input class="btn btn-danger btn-xs" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
-            </form>
-          </td>
-          <td>
-            <a href="<%= root_path %>unique_digests/<%= digest %>"><%= digest %></a>
-          </td>
-        </tr>
-      <% end %>
-    </table>
-
-    <form action="<%= root_path %>unique_digests/delete_all" method="get">
-      <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
-    </form>
-  </div>
-<% end %>

data/redis/acquire_lock.lua

@@ -1,21 +0,0 @@
-local unique_key = KEYS[1]
-local job_id     = ARGV[1]
-local expires    = tonumber(ARGV[2])
-local stored_jid = redis.pcall('get', unique_key)
-
-if stored_jid then
-  if stored_jid == job_id then
-    return 1
-  else
-    return 0
-  end
-end
-
-if redis.call('SET', unique_key, job_id, 'nx') then
-  if expires then
-    redis.call('EXPIRE', unique_key, expires)
-  end
-  return 1
-else
-  return 0
-end

data/redis/convert_legacy_lock.lua

@@ -1,13 +0,0 @@
-local grabbed_key   = KEYS[1]
-local unique_digest = KEYS[2]
-
-local job_id = ARGV[1]
-local current_time = tonumber(ARGV[2])
-
-local old_token  = redis.call('GET', unique_digest)
-if old_token then
-  if old_token == job_id or old_token == '2' then
-    redis.call('DEL', unique_digest)
-    redis.call('HSET', grabbed_key, job_id, current_time)
-  end
-end

data/redis/delete.lua

@@ -1,14 +0,0 @@
-local exists_key    = KEYS[1]
-local grabbed_key   = KEYS[2]
-local available_key = KEYS[3]
-local version_key   = KEYS[4]
-local unique_keys   = KEYS[5]
-local unique_digest = KEYS[6]     -- TODO: Legacy support (Remove in v6.1)
-
-redis.call('DEL', exists_key)
-redis.call('SREM', unique_keys, unique_digest)
-redis.call('DEL', grabbed_key)
-redis.call('DEL', available_key)
-redis.call('DEL', version_key)
-redis.call('DEL', 'uniquejobs')   -- TODO: Old job hash, just drop the darn thing
-redis.call('DEL', unique_digest)  -- TODO: Legacy support (Remove in v6.1)