sidekiq-unique-jobs 5.0.11 → 6.0.0.rc1

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -1,15 +1,20 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  module Lock
-    class UntilAndWhileExecuting < UntilExecuting
+  class Lock
+    class UntilAndWhileExecuting < BaseLock
       def execute(callback)
-        lock = WhileExecuting.new(item, redis_pool)
-        lock.synchronize do
-          callback.call if unlock(:server)
-          yield
+        return unless locked?
+        delete!
+
+        runtime_lock.execute(callback) do
+          yield if block_given?
         end
       end
+
+      def runtime_lock
+        @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item, redis_pool)
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -1,68 +1,16 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  module Lock
-    class UntilExecuted
+  class Lock
+    class UntilExecuted < BaseLock
       OK ||= 'OK'
 
-      include SidekiqUniqueJobs::Unlockable
-
-      extend Forwardable
-      def_delegators :Sidekiq, :logger
-
-      def initialize(item, redis_pool = nil)
-        @item = item
-        @redis_pool = redis_pool
-      end
-
-      def execute(callback, &blk)
-        operative = true
-        send(:after_yield_yield, &blk)
-      rescue Sidekiq::Shutdown
-        operative = false
-        raise
-      ensure
-        if operative && unlock(:server)
-          callback.call
-        else
-          logger.fatal { "the unique_key: #{unique_key} needs to be unlocked manually" }
-        end
-      end
-
-      def unlock(scope)
-        unless [:server, :api, :test].include?(scope)
-          raise ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
+      def execute(callback)
+        return unless locked?
+        using_protection(callback) do
+          yield if block_given?
         end
-
-        unlock_by_key(unique_key, item[JID_KEY], redis_pool)
       end
-
-      def lock(scope)
-        raise ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}" if scope.to_sym != :client
-
-        Scripts::AcquireLock.execute(
-          redis_pool,
-          unique_key,
-          item[JID_KEY],
-          max_lock_time,
-        )
-      end
-
-      def unique_key
-        @unique_key ||= UniqueArgs.digest(item)
-      end
-
-      def max_lock_time
-        @max_lock_time ||= QueueLockTimeoutCalculator.for_item(item).seconds
-      end
-
-      def after_yield_yield
-        yield
-      end
-
-      private
-
-      attr_reader :item, :redis_pool
     end
   end
 end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  module Lock
-    class UntilExecuting < UntilExecuted
-      def execute(callback, &_block)
-        callback.call if unlock(:server)
-        yield
+  class Lock
+    class UntilExecuting < BaseLock
+      def execute(callback)
+        delete
+        callback.call
+        yield if block_given?
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    class UntilExpired < BaseLock
+      def unlock
+        true
+      end
+
+      def execute(callback)
+        return unless locked?
+        yield if block_given?
+        callback.call
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -1,50 +1,36 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  module Lock
-    class WhileExecuting
-      def self.synchronize(item, redis_pool = nil)
-        new(item, redis_pool).synchronize { yield }
-      end
+  class Lock
+    class WhileExecuting < BaseLock
+      RUN_SUFFIX ||= ':RUN'
 
       def initialize(item, redis_pool = nil)
-        @item = item
-        @redis_pool = redis_pool
-        @unique_digest = "#{create_digest}:run"
-        @mutex = Mutex.new
-      end
-
-      def synchronize
-        @mutex.synchronize do
-          sleep 0.1 until locked?
-          yield
-        end
-      rescue Sidekiq::Shutdown
-        logger.fatal { "the unique_key: #{@unique_digest} needs to be unlocked manually" }
-        raise
-      ensure
-        SidekiqUniqueJobs.connection(@redis_pool) { |conn| conn.del @unique_digest }
+        super(item, redis_pool)
+        append_unique_key_suffix
       end
 
-      def locked?
-        Scripts.call(:synchronize, @redis_pool,
-                     keys: [@unique_digest],
-                     argv: [Time.now.to_i, max_lock_time]) == 1
+      # Returning true makes sure the client
+      #   can push the job on the queue
+      def lock
+        true
       end
 
-      def max_lock_time
-        @max_lock_time ||= RunLockTimeoutCalculator.for_item(@item).seconds
-      end
+      # Locks the job with the RUN_SUFFIX appended
+      def execute(callback)
+        return unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
 
-      def execute(_callback)
-        synchronize do
-          yield
+        using_protection(callback) do
+          yield if block_given?
         end
       end
 
-      def create_digest
-        @create_digest ||= @item[UNIQUE_DIGEST_KEY]
-        @create_digest ||= SidekiqUniqueJobs::UniqueArgs.digest(@item)
+      private
+
+      # This is safe as the base_lock always creates a new digest
+      #   The append there for needs to be done every time
+      def append_unique_key_suffix
+        item[UNIQUE_DIGEST_KEY] = item[UNIQUE_DIGEST_KEY] + RUN_SUFFIX
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    class WhileExecutingReject < WhileExecuting
+      def execute(callback)
+        return reject unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
+
+        using_protection(callback) do
+          yield if block_given?
+        end
+      end
+
+      # Private below here, keeping public due to testing reasons
+
+      def reject
+        log_debug { "Rejecting job with jid: #{item[JID_KEY]} already running" }
+        send_to_deadset
+      end
+
+      def send_to_deadset
+        log_info { "Adding dead #{item[CLASS_KEY]} job #{item[JID_KEY]}" }
+
+        if deadset_kill?
+          deadset_kill
+        else
+          push_to_deadset
+        end
+      end
+
+      def deadset_kill?
+        deadset.respond_to?(:kill)
+      end
+
+      def deadset_kill
+        if kill_with_options?
+          kill_job_with_options
+        else
+          kill_job_without_options
+        end
+      end
+
+      def kill_with_options?
+        Sidekiq::DeadSet.instance_method(:kill).arity > 1
+      end
+
+      def kill_job_without_options
+        deadset.kill(payload)
+      end
+
+      def kill_job_with_options
+        deadset.kill(payload, notify_failure: false)
+      end
+
+      def deadset
+        @deadset ||= Sidekiq::DeadSet.new
+      end
+
+      def push_to_deadset
+        Sidekiq.redis do |conn|
+          conn.multi do
+            conn.zadd('dead', current_time, payload)
+            conn.zremrangebyscore('dead', '-inf', current_time - Sidekiq::DeadSet.timeout)
+            conn.zremrangebyrank('dead', 0, -Sidekiq::DeadSet.max_jobs)
+          end
+        end
+      end
+
+      def current_time
+        @current_time ||= Time.now.to_f
+      end
+
+      def payload
+        @payload ||= Sidekiq.dump_json(item)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/while_executing_requeue.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Lock
+    class WhileExecutingRequeue < WhileExecuting
+      def lock
+        true
+      end
+
+      def execute(callback)
+        locksmith.lock(item[LOCK_TIMEOUT_KEY], raise: true) do
+          yield
+          callback.call
+        end
+
+        Sidekiq::Client.push(item) unless locksmith.locked?
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  class Locksmith # rubocop:disable ClassLength
+    API_VERSION = '1'
+    EXPIRES_IN = 10
+
+    include SidekiqUniqueJobs::Connection
+
+    def initialize(item, redis_pool = nil)
+      @concurrency   = 1 # removed in a0cff5bc42edbe7190d6ede7e7f845074d2d7af6
+      @expiration    = item[LOCK_EXPIRATION_KEY]
+      @jid           = item[JID_KEY]
+      @unique_digest = item[UNIQUE_DIGEST_KEY]
+      @redis_pool    = redis_pool
+    end
+
+    def create
+      Scripts.call(
+        :create,
+        redis_pool,
+        keys: [exists_key, grabbed_key, available_key, version_key, unique_digest],
+        argv: [jid, expiration, API_VERSION, concurrency],
+      )
+    end
+
+    def exists?
+      redis(redis_pool) { |conn| conn.exists(exists_key) }
+    end
+
+    def available_count
+      return concurrency unless exists?
+
+      redis(redis_pool) { |conn| conn.llen(available_key) }
+    end
+
+    def delete
+      return unless expiration.nil?
+      delete!
+    end
+
+    def delete!
+      Scripts.call(
+        :delete,
+        redis_pool,
+        keys: [exists_key, grabbed_key, available_key, version_key, unique_digest],
+      )
+    end
+
+    def lock(timeout = nil, &block)
+      create
+
+      grab_token(timeout) do |token|
+        touch_grabbed_token(token)
+        return_token_or_block_value(token, &block)
+      end
+    end
+    alias wait lock
+
+    def unlock
+      return false unless locked?
+      signal(jid)
+    end
+
+    def locked?(token = nil)
+      token ||= jid
+      redis(redis_pool) { |conn| conn.hexists(grabbed_key, token) }
+    end
+
+    def signal(token = nil)
+      token ||= jid
+
+      Scripts.call(
+        :signal,
+        redis_pool,
+        keys: [exists_key, grabbed_key, available_key, version_key, unique_digest],
+        argv: [token, expiration],
+      )
+    end
+
+    private
+
+    attr_reader :concurrency, :unique_digest, :expiration, :jid, :redis_pool
+
+    def grab_token(timeout = nil)
+      redis(redis_pool) do |conn|
+        if timeout.nil? || timeout.positive?
+          # passing timeout 0 to blpop causes it to block
+          _key, token = conn.blpop(available_key, timeout || 0)
+        else
+          token = conn.lpop(available_key)
+        end
+        yield token if token == jid
+      end
+    end
+
+    def touch_grabbed_token(token)
+      redis(redis_pool) { |conn| conn.hset(grabbed_key, token, current_time.to_f) }
+    end
+
+    def return_token_or_block_value(token)
+      return token unless block_given?
+
+      begin
+        yield token
+      ensure
+        signal(token)
+      end
+    end
+
+    def available_key
+      @available_key ||= namespaced_key('AVAILABLE')
+    end
+
+    def exists_key
+      @exists_key ||= namespaced_key('EXISTS')
+    end
+
+    def grabbed_key
+      @grabbed_key ||= namespaced_key('GRABBED')
+    end
+
+    def version_key
+      @version_key ||= namespaced_key('VERSION')
+    end
+
+    def namespaced_key(variable)
+      "#{unique_digest}:#{variable}"
+    end
+
+    def current_time
+      seconds, microseconds_with_frac = redis_time
+      Time.at(seconds, microseconds_with_frac)
+    end
+
+    def redis_time
+      redis(&:time)
+    end
+  end
+end
+
+require 'sidekiq_unique_jobs/lock/base_lock'
+require 'sidekiq_unique_jobs/lock/until_executed'
+require 'sidekiq_unique_jobs/lock/until_executing'
+require 'sidekiq_unique_jobs/lock/until_expired'
+require 'sidekiq_unique_jobs/lock/while_executing'
+require 'sidekiq_unique_jobs/lock/while_executing_reject'
+require 'sidekiq_unique_jobs/lock/while_executing_requeue'
+require 'sidekiq_unique_jobs/lock/until_and_while_executing'