activejob-unique 0.5.0

data/lib/active_job/uniqueness/lock_manager.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    # Redlock requires a value of the lock to release the resource by Redlock::Client#unlock method.
+    # LockManager introduces LockManager#delete_lock to unlock by resource key only.
+    # See https://github.com/leandromoreira/redlock-rb/issues/51 for more details.
+    class LockManager < ::Redlock::Client
+      # Unlocks a resource by resource only.
+      def delete_lock(resource)
+        @servers.each do |server|
+          synced_redis_connection(server) do |conn|
+            conn.call('DEL', resource)
+          end
+        end
+
+        true
+      end
+
+      DELETE_LOCKS_SCAN_COUNT = 1000
+
+      # Unlocks multiple resources by key wildcard.
+      def delete_locks(wildcard)
+        @servers.each do |server|
+          synced_redis_connection(server) do |conn|
+            cursor = 0
+            while cursor != '0'
+              cursor, keys = conn.call('SCAN', cursor, 'MATCH', wildcard, 'COUNT', DELETE_LOCKS_SCAN_COUNT)
+              conn.call('UNLINK', *keys) unless keys.empty?
+            end
+          end
+        end
+
+        true
+      end
+
+      private
+
+      def synced_redis_connection(server, &block)
+        if server.respond_to?(:synchronize)
+          server.synchronize(&block)
+        else
+          server.instance_variable_get(:@redis).with(&block)
+        end
+      end
+    end
+  end
+end

data/lib/active_job/uniqueness/log_subscriber.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'active_support/log_subscriber'
+
+module ActiveJob
+  module Uniqueness
+    class LogSubscriber < ActiveSupport::LogSubscriber # :nodoc:
+      def lock(event)
+        job = event.payload[:job]
+        resource = event.payload[:resource]
+
+        debug do
+          "Locked #{lock_info(job, resource)}" + args_info(job)
+        end
+      end
+
+      def runtime_lock(event)
+        job = event.payload[:job]
+        resource = event.payload[:resource]
+
+        debug do
+          "Locked runtime #{lock_info(job, resource)}" + args_info(job)
+        end
+      end
+
+      def unlock(event)
+        job = event.payload[:job]
+        resource = event.payload[:resource]
+
+        debug do
+          "Unlocked #{lock_info(job, resource)}"
+        end
+      end
+
+      def runtime_unlock(event)
+        job = event.payload[:job]
+        resource = event.payload[:resource]
+
+        debug do
+          "Unlocked runtime #{lock_info(job, resource)}"
+        end
+      end
+
+      def conflict(event)
+        job = event.payload[:job]
+        resource = event.payload[:resource]
+
+        info do
+          "Not unique #{lock_info(job, resource)}" + args_info(job)
+        end
+      end
+
+      def runtime_conflict(event)
+        job = event.payload[:job]
+        resource = event.payload[:resource]
+
+        info do
+          "Not unique runtime #{lock_info(job, resource)}" + args_info(job)
+        end
+      end
+
+      private
+
+      def lock_info(job, resource)
+        "#{job.class.name} (Job ID: #{job.job_id}) (Lock key: #{resource})"
+      end
+
+      def args_info(job)
+        if job.arguments.any? && log_arguments?(job)
+          " with arguments: #{job.arguments.map { |arg| format(arg).inspect }.join(', ')}"
+        else
+          ''
+        end
+      end
+
+      def log_arguments?(job)
+        return true unless job.class.respond_to?(:log_arguments?)
+
+        job.class.log_arguments?
+      end
+
+      def format(arg)
+        case arg
+        when Hash
+          arg.transform_values { |value| format(value) }
+        when Array
+          arg.map { |value| format(value) }
+        when GlobalID::Identification
+          arg.to_global_id rescue arg
+        else
+          arg
+        end
+      end
+
+      def logger
+        ActiveJob::Base.logger
+      end
+    end
+  end
+end
+
+ActiveJob::Uniqueness::LogSubscriber.attach_to :active_job_uniqueness

data/lib/active_job/uniqueness/sidekiq_patch.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'activejob/uniqueness'
+require 'sidekiq/api'
+
+module ActiveJob
+  module Uniqueness
+    SIDEKIQ_JOB_WRAPPERS = %w[
+      ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper
+      Sidekiq::ActiveJob::Wrapper
+    ].freeze
+
+    def self.unlock_sidekiq_job!(job_data)
+      return unless SIDEKIQ_JOB_WRAPPERS.include?(job_data['class'])
+
+      job = ActiveJob::Base.deserialize(job_data.fetch('args').first)
+
+      return unless job.class.lock_strategy_class
+
+      begin
+        job.send(:deserialize_arguments_if_needed)
+      rescue ActiveJob::DeserializationError
+        # Most probably, GlobalID fails to locate AR record (record is deleted)
+      else
+        ActiveJob::Uniqueness.unlock!(job_class_name: job.class.name, arguments: job.arguments)
+      end
+    end
+
+    module SidekiqPatch
+      module SortedEntry
+        def delete
+          ActiveJob::Uniqueness.unlock_sidekiq_job!(item) if super
+          item
+        end
+
+        private
+
+        def remove_job
+          super do |message|
+            ActiveJob::Uniqueness.unlock_sidekiq_job!(Sidekiq.load_json(message))
+            yield message
+          end
+        end
+      end
+
+      module ScheduledSet
+        def delete(score, job_id)
+          entry = find_job(job_id)
+          ActiveJob::Uniqueness.unlock_sidekiq_job!(entry.item) if super
+          entry
+        end
+      end
+
+      module Job
+        def delete
+          ActiveJob::Uniqueness.unlock_sidekiq_job!(item)
+          super
+        end
+      end
+
+      module Queue
+        def clear
+          each(&:delete)
+          super
+        end
+      end
+
+      module JobSet
+        def clear
+          each(&:delete)
+          super
+        end
+
+        def delete_by_value(name, value)
+          ActiveJob::Uniqueness.unlock_sidekiq_job!(Sidekiq.load_json(value)) if super
+        end
+      end
+    end
+  end
+end
+
+Sidekiq::SortedEntry.prepend ActiveJob::Uniqueness::SidekiqPatch::SortedEntry
+Sidekiq::ScheduledSet.prepend ActiveJob::Uniqueness::SidekiqPatch::ScheduledSet
+Sidekiq::Queue.prepend ActiveJob::Uniqueness::SidekiqPatch::Queue
+Sidekiq::JobSet.prepend ActiveJob::Uniqueness::SidekiqPatch::JobSet
+
+sidekiq_version = Gem::Version.new(Sidekiq::VERSION)
+
+# Sidekiq 6.2.2 renames Sidekiq::Job to Sidekiq::JobRecord
+# https://github.com/mperham/sidekiq/issues/4955
+if sidekiq_version >= Gem::Version.new('6.2.2')
+  Sidekiq::JobRecord.prepend ActiveJob::Uniqueness::SidekiqPatch::Job
+else
+  Sidekiq::Job.prepend ActiveJob::Uniqueness::SidekiqPatch::Job
+end
+
+# Global death handlers are introduced in Sidekiq 5.1
+# https://github.com/mperham/sidekiq/blob/e7acb124fbeb0bece0a7c3d657c39a9cc18d72c6/Changes.md#510
+if sidekiq_version >= Gem::Version.new('7.0')
+  Sidekiq.default_configuration.death_handlers << ->(job, _ex) { ActiveJob::Uniqueness.unlock_sidekiq_job!(job) }
+elsif sidekiq_version >= Gem::Version.new('5.1')
+  Sidekiq.death_handlers << ->(job, _ex) { ActiveJob::Uniqueness.unlock_sidekiq_job!(job) }
+end

data/lib/active_job/uniqueness/strategies/base.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    module Strategies
+      # Base strategy is not supposed to actually be used as uniqueness strategy.
+      class Base
+        # https://github.com/rails/rails/pull/17227
+        # https://groups.google.com/g/rubyonrails-core/c/mhD4T90g0G4
+        ACTIVEJOB_SUPPORTS_THROW_ABORT = ActiveJob.gem_version >= Gem::Version.new('5.0')
+
+        delegate :lock_manager, :config, to: :'ActiveJob::Uniqueness'
+
+        attr_reader :lock_key, :lock_ttl, :on_conflict, :on_redis_connection_error, :job
+
+        def initialize(job:)
+          @lock_key = job.lock_key
+          @lock_ttl = (job.lock_options[:lock_ttl] || config.lock_ttl).to_i * 1000 # ms
+          @on_conflict = job.lock_options[:on_conflict] || config.on_conflict
+          @on_redis_connection_error = job.lock_options[:on_redis_connection_error] || config.on_redis_connection_error
+          @job = job
+        end
+
+        def lock(resource:, ttl:, event: :lock)
+          lock_manager.lock(resource, ttl).tap do |result|
+            instrument(event, resource: resource, ttl: ttl) if result
+          end
+        end
+
+        def unlock(resource:, event: :unlock)
+          lock_manager.delete_lock(resource).tap do
+            instrument(event, resource: resource)
+          end
+        end
+
+        def before_enqueue
+          # Expected to be overriden in the descendant strategy
+        end
+
+        def before_perform
+          # Expected to be overriden in the descendant strategy
+        end
+
+        def around_enqueue(block)
+          # Expected to be overriden in the descendant strategy
+          block.call
+        end
+
+        def around_perform(block)
+          # Expected to be overriden in the descendant strategy
+          block.call
+        end
+
+        def after_perform
+          # Expected to be overriden in the descendant strategy
+        end
+
+        module LockingOnEnqueue
+          def before_enqueue
+            return if lock(resource: lock_key, ttl: lock_ttl)
+
+            handle_conflict(resource: lock_key, on_conflict: on_conflict)
+            abort_job
+          rescue RedisClient::ConnectionError => e
+            handle_redis_connection_error(
+              resource: lock_key, on_redis_connection_error:
+              on_redis_connection_error, error: e
+            )
+            abort_job
+          end
+
+          def around_enqueue(block)
+            return if @job_aborted # ActiveJob 4.2 workaround
+
+            enqueued = false
+
+            block.call
+
+            enqueued = true
+          ensure
+            unlock(resource: lock_key) unless @job_aborted || enqueued
+          end
+        end
+
+        private
+
+        def handle_conflict(on_conflict:, resource:, event: :conflict)
+          case on_conflict
+          when :log then instrument(event, resource: resource)
+          when :raise then raise_not_unique_job_error(resource: resource, event: event)
+          else
+            on_conflict.call(job)
+          end
+        end
+
+        def handle_redis_connection_error(resource:, on_redis_connection_error:, error:)
+          case on_redis_connection_error
+          when :raise, nil then raise error
+          else
+            on_redis_connection_error.call(job, resource: resource, error: error)
+          end
+        end
+
+        def abort_job
+          @job_aborted = true # ActiveJob 4.2 workaround
+
+          ACTIVEJOB_SUPPORTS_THROW_ABORT ? throw(:abort) : false
+        end
+
+        def instrument(action, payload = {})
+          ActiveSupport::Notifications.instrument "#{action}.active_job_uniqueness", payload.merge(job: job)
+        end
+
+        def raise_not_unique_job_error(resource:, event:)
+          message = [
+            job.class.name,
+            "(Job ID: #{job.job_id})",
+            "(Lock key: #{resource})",
+            job.arguments.inspect
+          ]
+
+          message.unshift(event == :runtime_conflict ? 'Not unique runtime' : 'Not unique')
+
+          raise JobNotUnique, message.join(' ')
+        end
+      end
+    end
+  end
+end

data/lib/active_job/uniqueness/strategies/until_and_while_executing.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    module Strategies
+      # Locks the job when it is pushed to the queue.
+      # Unlocks the job before the job is started.
+      # Then creates runtime lock to prevent simultaneous jobs from being executed.
+      class UntilAndWhileExecuting < Base
+        include LockingOnEnqueue
+
+        attr_reader :runtime_lock_key, :runtime_lock_ttl, :on_runtime_conflict
+
+        def initialize(job:)
+          super
+          @runtime_lock_key = job.runtime_lock_key
+
+          runtime_lock_ttl_option = job.lock_options[:runtime_lock_ttl]
+          @runtime_lock_ttl = runtime_lock_ttl_option.present? ? runtime_lock_ttl_option.to_i * 1000 : lock_ttl
+
+          @on_runtime_conflict = job.lock_options[:on_runtime_conflict] || on_conflict
+        end
+
+        def before_perform
+          unlock(resource: lock_key)
+
+          return if lock(resource: runtime_lock_key, ttl: runtime_lock_ttl, event: :runtime_lock)
+
+          handle_conflict(on_conflict: on_runtime_conflict, resource: runtime_lock_key, event: :runtime_conflict)
+          abort_job
+        end
+
+        def around_perform(block)
+          return if @job_aborted # ActiveJob 4.2 workaround
+
+          block.call
+        ensure
+          unlock(resource: runtime_lock_key, event: :runtime_unlock) unless @job_aborted
+        end
+      end
+    end
+  end
+end

data/lib/active_job/uniqueness/strategies/until_executed.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    module Strategies
+      # Locks the job when it is pushed to the queue.
+      # Unlocks the job when the job is finished.
+      class UntilExecuted < Base
+        include LockingOnEnqueue
+
+        def after_perform
+          unlock(resource: lock_key)
+        end
+      end
+    end
+  end
+end

data/lib/active_job/uniqueness/strategies/until_executing.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    module Strategies
+      # Locks the job when it is pushed to the queue.
+      # Unlocks the job before the job is started.
+      class UntilExecuting < Base
+        include LockingOnEnqueue
+
+        def before_perform
+          unlock(resource: lock_key)
+        end
+      end
+    end
+  end
+end

data/lib/active_job/uniqueness/strategies/until_expired.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    module Strategies
+      # Locks the job when it is pushed to the queue.
+      # Does not allow new jobs enqueued until lock is expired.
+      class UntilExpired < Base
+        include LockingOnEnqueue
+      end
+    end
+  end
+end

data/lib/active_job/uniqueness/strategies/while_executing.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    module Strategies
+      # Locks the job when the job starts.
+      # Unlocks the job when the job is finished.
+      class WhileExecuting < Base
+        def before_perform
+          return if lock(resource: lock_key, ttl: lock_ttl, event: :runtime_lock)
+
+          handle_conflict(resource: lock_key, event: :runtime_conflict, on_conflict: on_conflict)
+          abort_job
+        end
+
+        def around_perform(block)
+          return if @job_aborted # ActiveJob 4.2 workaround
+
+          block.call
+        ensure
+          unlock(resource: lock_key, event: :runtime_unlock) unless @job_aborted
+        end
+      end
+    end
+  end
+end

data/lib/active_job/uniqueness/strategies.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    # See Configuration#lock_strategies if you want to define custom strategy
+    module Strategies
+      extend ActiveSupport::Autoload
+
+      autoload :Base
+      autoload :UntilExpired
+      autoload :UntilExecuted
+      autoload :UntilExecuting
+      autoload :UntilAndWhileExecuting
+      autoload :WhileExecuting
+
+      class << self
+        def lookup(strategy)
+          matching_strategy(strategy.to_s.camelize) ||
+            ActiveJob::Uniqueness.config.lock_strategies[strategy] ||
+            raise(StrategyNotFound, "Strategy '#{strategy}' is not found. Is it declared in the configuration?")
+        end
+
+        private
+
+        def matching_strategy(const)
+          const_get(const, false) if const_defined?(const, false)
+        end
+      end
+    end
+  end
+end

data/lib/active_job/uniqueness/test_lock_manager.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    # Mocks ActiveJob::Uniqueness::LockManager methods.
+    # See ActiveJob::Uniqueness.test_mode!
+    class TestLockManager
+      def lock(*_args)
+        true
+      end
+
+      alias delete_lock lock
+      alias delete_locks lock
+    end
+  end
+end

data/lib/active_job/uniqueness/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    VERSION = '0.5.0'
+  end
+end

data/lib/active_job/uniqueness.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'active_job'
+require 'redlock'
+
+require 'active_job/uniqueness/version'
+require 'active_job/uniqueness/errors'
+require 'active_job/uniqueness/log_subscriber'
+require 'active_job/uniqueness/active_job_patch'
+
+module ActiveJob
+  module Uniqueness
+    extend ActiveSupport::Autoload
+
+    autoload :Configuration
+    autoload :LockKey
+    autoload :Strategies
+    autoload :LockManager
+    autoload :TestLockManager
+
+    class << self
+      def configure
+        yield config
+      end
+
+      def config
+        @config ||= ActiveJob::Uniqueness::Configuration.new
+      end
+
+      def lock_manager
+        @lock_manager ||= ActiveJob::Uniqueness::LockManager.new(config.redlock_servers, config.redlock_options)
+      end
+
+      def unlock!(**args)
+        lock_manager.delete_locks(ActiveJob::Uniqueness::LockKey.new(**args).wildcard_key)
+      end
+
+      def test_mode!
+        @lock_manager = ActiveJob::Uniqueness::TestLockManager.new
+      end
+
+      def reset_manager!
+        @lock_manager = nil
+      end
+    end
+  end
+end

data/lib/activejob/uniqueness.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require 'active_job/uniqueness'

data/lib/activejob-unique.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+# This file intentionally uses a dash to match the gem name 'activejob-unique'
+require 'active_job/uniqueness'

data/lib/generators/active_job/uniqueness/install_generator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    module Generators
+      class InstallGenerator < Rails::Generators::Base
+        desc 'Copy ActiveJob::Uniqueness default files'
+        source_root File.expand_path('templates', __dir__)
+
+        def copy_config
+          template 'config/initializers/active_job_uniqueness.rb'
+        end
+      end
+    end
+  end
+end

data/lib/generators/active_job/uniqueness/templates/config/initializers/active_job_uniqueness.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+ActiveJob::Uniqueness.configure do |config|
+  # Global default expiration for lock keys. Each job can define its own ttl via :lock_ttl option.
+  # Strategy :until_and_while_executing also accepts :on_runtime_ttl option.
+  #
+  # config.lock_ttl = 1.day
+
+  # Prefix for lock keys. Can not be set per job.
+  #
+  # config.lock_prefix = 'activejob_uniqueness'
+
+  # Default action on lock conflict. Can be set per job.
+  # Strategy :until_and_while_executing also accepts :on_runtime_conflict option.
+  # Allowed values are
+  #   :raise - raises ActiveJob::Uniqueness::JobNotUnique
+  #   :log - instruments ActiveSupport::Notifications and logs event to the ActiveJob::Logger
+  #   proc - custom Proc. For example, ->(job) { job.logger.info("Job already in queue: #{job.class.name} #{job.arguments.inspect} (#{job.job_id})") }
+  #
+  # config.on_conflict = :raise
+
+  # Default action on redis connection error. Can be set per job.
+  # Allowed values are
+  #   :raise - raises ActiveJob::Uniqueness::JobNotUnique
+  #   proc - custom Proc. For example, ->(job, resource: _, error: _) { job.logger.info("Job already in queue: #{job.class.name} #{job.arguments.inspect} (#{job.job_id})") }
+  #
+  # config.on_redis_connection_error = :raise
+
+  # Digest method for lock keys generating. Expected to have `hexdigest` class method.
+  #
+  # config.digest_method = OpenSSL::Digest::MD5
+
+  # Array of redis servers for Redlock quorum.
+  # Read more at https://github.com/leandromoreira/redlock-rb#redis-client-configuration
+  #
+  # config.redlock_servers = [ENV.fetch('REDIS_URL', 'redis://localhost:6379')]
+
+  # Custom options for Redlock.
+  # Read more at https://github.com/leandromoreira/redlock-rb#redlock-configuration
+  #
+  # config.redlock_options = { retry_count: 0 }
+
+  # Custom strategies.
+  # config.lock_strategies = { my_strategy: MyStrategy }
+  #
+  # config.lock_strategies = {}
+end