sidekiq-unique-jobs 3.0.14 → 4.0.0

data/lib/sidekiq_unique_jobs/client/middleware.rb

@@ -0,0 +1,47 @@
+require 'sidekiq_unique_jobs/server/middleware'
+
+module SidekiqUniqueJobs
+  module Client
+    class Middleware
+      extend Forwardable
+      def_delegators :SidekiqUniqueJobs, :connection, :config
+      def_delegators :Sidekiq, :logger
+
+      include OptionsWithFallback
+
+      def call(worker_class, item, queue, redis_pool = nil)
+        @worker_class = SidekiqUniqueJobs.worker_class_constantize(worker_class)
+        @item = item
+        @queue = queue
+        @redis_pool = redis_pool
+        yield if ordinary_or_locked?
+      end
+
+      private
+
+      attr_reader :item, :worker_class, :redis_pool, :queue
+
+      def ordinary_or_locked?
+        unique_disabled? || unlockable? || aquire_lock
+      end
+
+      def unlockable?
+        !lockable?
+      end
+
+      def lockable?
+        lock.respond_to?(:lock)
+      end
+
+      def aquire_lock
+        locked = lock.lock(:client)
+        warn_about_duplicate(item) unless locked
+        locked
+      end
+
+      def warn_about_duplicate(item)
+        logger.warn "payload is not unique #{item}" if log_duplicate_payload?
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/config.rb

@@ -1,47 +1,23 @@
 module SidekiqUniqueJobs
   class Config < OpenStruct
+    TESTING_CONSTANT ||= 'Testing'.freeze
     CONFIG_ACCESSORS = [
       :unique_prefix,
-      :unique_args_enabled,
       :default_expiration,
-      :default_unlock_order,
-      :unique_storage_method
-    ]
-
-    class << self
-      CONFIG_ACCESSORS.each do |method|
-        define_method(method) do
-          warn("#{method} has been deprecated. See readme for information")
-          config.send(method)
-        end
-
-        define_method("#{method}=") do |obj|
-          warn("#{method} has been deprecated. See readme for information")
-          config.send("#{method}=", obj)
-        end
-      end
-
-      def unique_args_enabled?
-        warn('unique_args_enabled has been deprecated. See readme for information')
-        config.unique_args_enabled
-      end
-
-      def config
-        SidekiqUniqueJobs.config
-      end
-    end
+      :default_lock,
+      :redis_mode
+    ].freeze
 
     def inline_testing_enabled?
-      if testing_enabled? && Sidekiq::Testing.inline?
-        require 'sidekiq_unique_jobs/inline_testing'
-        return true
-      end
+      testing_enabled? && Sidekiq::Testing.inline?
+    end
 
-      false
+    def mocking?
+      inline_testing_enabled? && redis_test_mode.to_sym == :mock
     end
 
     def testing_enabled?
-      Sidekiq.const_defined?('Testing') && Sidekiq::Testing.enabled?
+      Sidekiq.const_defined?(TESTING_CONSTANT) && Sidekiq::Testing.enabled?
     end
 
     def unique_args_enabled?

data/lib/sidekiq_unique_jobs/core_ext.rb

@@ -0,0 +1,46 @@
+begin
+  require 'active_support/core_ext/hash/keys'
+  require 'active_support/core_ext/hash/deep_merge'
+rescue LoadError
+  class Hash
+    def slice(*keys)
+      keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
+      keys.each_with_object(self.class.new) { |k, hash| hash[k] = self[k] if key?(k) }
+    end unless {}.respond_to?(:slice)
+
+    def slice!(*keys)
+      keys.map! { |key| convert_key(key) } if respond_to?(:convert_key, true)
+      omit = slice(*self.keys - keys)
+      hash = slice(*keys)
+      hash.default      = default
+      hash.default_proc = default_proc if default_proc
+      replace(hash)
+      omit
+    end unless {}.respond_to?(:slice!)
+  end
+end
+begin
+  require 'active_support/core_ext/string/inflections'
+rescue LoadError
+  class String
+    # File activesupport/lib/active_support/inflector/methods.rb, line 178
+    def classify
+      camelize(singularize(sub(/.*\./, '')))
+    end unless ''.respond_to?(:classify)
+
+    # File activesupport/lib/active_support/inflector/methods.rb, line 67
+    def camelize(uppercase_first_letter = true)
+      string = self
+      if uppercase_first_letter
+        string = string.sub(/^[a-z\d]*/) { $&.capitalize }
+      else
+        string = string.sub(/^(?:(?=\b|[A-Z_])|\w)/) { $&.downcase }
+      end
+      string.gsub!(%r{(?:_|(\/))([a-z\d]*)}i) do
+        "#{Regexp.last_match(1)}#{Regexp.last_match(2).capitalize}"
+      end
+      string.gsub!(%r{/}, '::')
+      string
+    end unless ''.respond_to?(:camelize)
+  end
+end

data/lib/sidekiq_unique_jobs/lock.rb

@@ -0,0 +1,10 @@
+require 'sidekiq_unique_jobs/lock/time_calculator'
+require 'sidekiq_unique_jobs/lock/until_executed'
+require 'sidekiq_unique_jobs/lock/until_executing'
+require 'sidekiq_unique_jobs/lock/while_executing'
+require 'sidekiq_unique_jobs/lock/until_timeout'
+
+module SidekiqUniqueJobs
+  module Lock
+  end
+end

data/lib/sidekiq_unique_jobs/lock/time_calculator.rb

@@ -0,0 +1,44 @@
+module SidekiqUniqueJobs
+  module Lock
+    class TimeCalculator
+      def self.for_item(item)
+        new(item)
+      end
+
+      def initialize(item)
+        @item = item
+      end
+
+      def seconds
+        time_until_scheduled + unique_expiration
+      end
+
+      def time_until_scheduled
+        scheduled = item['at'.freeze]
+        return 0 unless scheduled
+        (Time.at(scheduled) - Time.now.utc).to_i
+      end
+
+      def unique_expiration
+        @unique_expiration ||=
+          (
+            worker_class_unique_expiration ||
+            SidekiqUniqueJobs.config.default_expiration
+          ).to_i
+      end
+
+      def worker_class_unique_expiration
+        return unless worker_class.respond_to?(:get_sidekiq_options)
+        worker_class.get_sidekiq_options['unique_expiration'.freeze]
+      end
+
+      def worker_class
+        @worker_class ||= SidekiqUniqueJobs.worker_class_constantize(item['class'.freeze])
+      end
+
+      private
+
+      attr_reader :item
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -0,0 +1,56 @@
+module SidekiqUniqueJobs
+  module Lock
+    class UntilExecuted
+      OK ||= 'OK'.freeze
+
+      extend Forwardable
+      def_delegators :Sidekiq, :logger
+
+      def initialize(item, redis_pool = nil)
+        @item = item
+        @redis_pool = redis_pool
+      end
+
+      def unlock(scope)
+        unless [:server, :api, :test].include?(scope)
+          fail ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
+        end
+        SidekiqUniqueJobs::Unlockable.unlock(unique_key, item['jid'.freeze], redis_pool)
+      end
+
+      # rubocop:disable MethodLength
+      def lock(scope)
+        if scope.to_sym != :client
+          fail ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
+        end
+
+        result = Scripts.call(:aquire_lock, redis_pool,
+                              keys: [unique_key],
+                              argv: [item['jid'.freeze], max_lock_time])
+        case result
+        when 1
+          logger.debug { "successfully locked #{unique_key} for #{max_lock_time} seconds" }
+          true
+        when 0
+          logger.debug { "failed to aquire lock for #{unique_key}" }
+          false
+        else
+          fail "#{__method__} returned an unexpected value (#{result})"
+        end
+      end
+      # rubocop:enable MethodLength
+
+      def unique_key
+        @unique_key ||= UniqueArgs.digest(item)
+      end
+
+      def max_lock_time
+        @max_lock_time ||= TimeCalculator.for_item(item).seconds
+      end
+
+      private
+
+      attr_reader :item, :redis_pool
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -0,0 +1,6 @@
+module SidekiqUniqueJobs
+  module Lock
+    class UntilExecuting < UntilExecuted
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/until_timeout.rb

@@ -0,0 +1,10 @@
+module SidekiqUniqueJobs
+  module Lock
+    class UntilTimeout < UntilExecuted
+      def unlock(scope)
+        return true if scope.to_sym == :server
+        fail ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -0,0 +1,31 @@
+module SidekiqUniqueJobs
+  module Lock
+    class WhileExecuting
+      def self.synchronize(item, redis_pool = nil, &block)
+        new(item, redis_pool).synchronize(&block)
+      end
+
+      def initialize(item, redis_pool = nil)
+        @unique_digest = item['unique_digest'.freeze]
+        @run_key = "#{@unique_digest}:run"
+        @mutex = Mutex.new
+        @redis_pool = redis_pool
+      end
+
+      def synchronize(&_block)
+        @mutex.lock
+        sleep 0.001 until locked?
+
+        yield
+
+      ensure
+        SidekiqUniqueJobs.connection(@redis_pool) { |c| c.del @run_key }
+        @mutex.unlock
+      end
+
+      def locked?
+        Scripts.call(:synchronize, @redis_pool, keys: [@run_key], argv: [Time.now.to_i]) == 1
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -1,19 +1,35 @@
 require 'sidekiq'
 
-Sidekiq.configure_server do |config|
-  config.server_middleware do |chain|
-    require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
-    chain.add SidekiqUniqueJobs::Middleware::Server::UniqueJobs
-  end
-  config.client_middleware do |chain|
-    require 'sidekiq_unique_jobs/middleware/client/unique_jobs'
-    chain.add SidekiqUniqueJobs::Middleware::Client::UniqueJobs
-  end
-end
+module SidekiqUniqueJobs
+  module Middleware
+    def self.extended(base)
+      base.class_eval do
+        configure_middleware
+      end
+    end
+
+    def configure_middleware
+      configure_server_middleware
+      configure_client_middleware
+    end
+
+    def configure_server_middleware
+      Sidekiq.configure_server do |config|
+        config.server_middleware do |chain|
+          require 'sidekiq_unique_jobs/server/middleware'
+          chain.add SidekiqUniqueJobs::Server::Middleware
+        end
+      end
+    end
 
-Sidekiq.configure_client do |config|
-  config.client_middleware do |chain|
-    require 'sidekiq_unique_jobs/middleware/client/unique_jobs'
-    chain.add SidekiqUniqueJobs::Middleware::Client::UniqueJobs
+    def configure_client_middleware
+      Sidekiq.configure_client do |config|
+        config.client_middleware do |chain|
+          require 'sidekiq_unique_jobs/client/middleware'
+          chain.add SidekiqUniqueJobs::Client::Middleware
+        end
+      end
+    end
   end
 end
+SidekiqUniqueJobs.send(:extend, SidekiqUniqueJobs::Middleware)

data/lib/sidekiq_unique_jobs/normalizer.rb

@@ -0,0 +1,7 @@
+module SidekiqUniqueJobs
+  module Normalizer
+    def self.jsonify(args)
+      JSON.parse(args.to_json)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -0,0 +1,36 @@
+module SidekiqUniqueJobs
+  module OptionsWithFallback
+    UNIQUE_KEY ||= 'unique'.freeze
+    UNIQUE_LOCK_KEY ||= 'unique_lock'.freeze
+    LOG_DUPLICATE_KEY ||= 'log_duplicate_payload'.freeze
+
+    def unique_enabled?
+      options[UNIQUE_KEY] || item[UNIQUE_KEY]
+    end
+
+    def unique_disabled?
+      !unique_enabled?
+    end
+
+    def log_duplicate_payload?
+      options[LOG_DUPLICATE_KEY] || item[LOG_DUPLICATE_KEY]
+    end
+
+    def lock
+      @lock = lock_class.new(item)
+    end
+
+    def lock_class
+      "SidekiqUniqueJobs::Lock::#{unique_lock.to_s.classify}".constantize
+    end
+
+    def unique_lock
+      options[UNIQUE_LOCK_KEY] || item[UNIQUE_LOCK_KEY] || SidekiqUniqueJobs.default_lock
+    end
+
+    def options
+      @options ||= worker_class.get_sidekiq_options if worker_class.respond_to?(:get_sidekiq_options)
+      @options ||= {}
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/run_lock_failed.rb

@@ -0,0 +1 @@
+SidekiqUniqueJobs::RunLockFailed = Class.new(StandardError)

data/lib/sidekiq_unique_jobs/scripts.rb

@@ -0,0 +1,50 @@
+require 'pathname'
+require 'digest/sha1'
+
+module SidekiqUniqueJobs
+  ScriptError = Class.new(StandardError)
+
+  module Scripts
+    extend Forwardable
+    LUA_PATHNAME ||= Pathname.new(__FILE__).dirname.join('../../redis').freeze
+    SOURCE_FILES ||= Dir[LUA_PATHNAME.join('**/*.lua')].compact.freeze
+    DEFINED_METHODS ||= []
+
+    module_function
+
+    def script_shas
+      @script_shas ||= {}
+    end
+
+    def logger
+      Sidekiq.logger
+    end
+
+    def call(file_name, redis_pool, options = {})
+      connection(redis_pool) do |redis|
+        script_shas[file_name] ||= redis.script(:load, script_source(file_name))
+        redis.evalsha(script_shas[file_name], options)
+      end
+    rescue Redis::CommandError => ex
+      raise ScriptError,
+            "#{file_name}.lua\n\n" +
+              ex.message + "\n\n" +
+              script_source(file_name) +
+              ex.backtrace.join("\n")
+    end
+
+    def connection(redis_pool, &_block)
+      SidekiqUniqueJobs.connection(redis_pool) do |conn|
+        yield conn
+      end
+    end
+
+    def script_source(file_name)
+      script_path(file_name).read
+    end
+
+    def script_path(file_name)
+      LUA_PATHNAME.join("#{file_name}.lua")
+    end
+  end
+end