sidekiq-unique-jobs 6.0.6 → 6.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f82a371b5f835c77c75c753533472d6b7654cb5f403ebeb686502046a0884681
-  data.tar.gz: 982ad72a4c7f7a164c9dadc7e5979b26562b53d7a934a6a04c2baf0bd35f7732
+  metadata.gz: 10e850f31b0ccdb3fb851cd075ab25ff6121245c7b0eace83e7075b637bbccf2
+  data.tar.gz: 6b6d7592acd4822c915b54c29d4fde9715709edbececdd7ff3324401ce8cfb8c
 SHA512:
-  metadata.gz: 6e6e9740d60600bdf0ab87b1c24dca5c680f866bbdee55449dcfe2dd143956b85ecf8db33c5bbdbc1c4ec59dcd9ab8ee481af06c93b234419612005ce1628f62
-  data.tar.gz: 58e048744c22ebf4b288863919dad091a122f9511b7d578b06327060e88e4c377542ebae00311381ec2cbda4276d8c6e08cf51033c6369532499aa2c2e49815b
+  metadata.gz: c68d91eaa41ab900ab2063a1a7f514283743f11546fff0ba479869c4479c8ff2ccb1b4d04636dcc6bb8f9dd8345562a09c488380dcadd5e1d1fb745eb5ec3564
+  data.tar.gz: a36b53f36718f17e2ab4f25d5456aae403e51a53cecdacd142c26dc28c177f147bce6c960db8b9ce31328c621ab0379251e27b0fed9b0de04d3175e23d8134e3

data/.gitignore

@@ -24,3 +24,5 @@ rails_example/spec/examples.txt
 /.byebug_history
 
 /.yardoc/
+
+/spec/examples.txt

data/.rubocop.yml

@@ -9,7 +9,7 @@ AllCops:
   Exclude:
     - 'Gemfile.lock'
     - 'gemfiles/**/*'
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 2.3
 
 Lint/HandleExceptions:
   Enabled: true

data/.travis.yml

@@ -8,13 +8,10 @@ dist: trusty
 sudo: required
 language: ruby
 cache: bundler
+services:
+  - redis-server
 
 before_install:
-  - sudo rm -rf /etc/apt/sources.list.d/rwky-redis.list
-  - sudo add-apt-repository ppa:chris-lea/redis-server -y
-  - sudo apt-get update -qy
-  - sudo apt-get install redis-server
-  - sudo service redis-server start
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter
 before_script:
@@ -42,6 +39,7 @@ gemfile:
   - gemfiles/sidekiq_4.2.gemfile
   - gemfiles/sidekiq_5.0.gemfile
   - gemfiles/sidekiq_5.1.gemfile
+  - gemfiles/sidekiq_5.2.gemfile
 
 notifications:
   email:

data/Appraisals

@@ -23,3 +23,7 @@ end
 appraise 'sidekiq-5.1' do
   gem 'sidekiq', '~> 5.1.0'
 end
+
+appraise 'sidekiq-5.2' do
+  gem 'sidekiq', '~> 5.2.0'
+end

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## v6.0.6
+
+- Fixes a bug with the command line utility ([#321](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/321))
+- Internal refactoring to improve performance ([#318](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/318))
+- Adds coverage for retrying jobs
+
+## v6.0.5
+
+- Fixes a bug with keys not being deleted ([#317](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/317))
+
 ## v6.0.4
 
 - Prevent locks from stealing each other (#316)

data/README.md

@@ -443,7 +443,7 @@ RSpec.describe Workers::CoolOne do
 
     it 'prevents duplicate jobs from being scheduled' do
       SidekiqUniqueJobs.use_config(enabled: true) do
-        expect(described_class.perform_async(1)).not_to eq(nil)
+        expect(described_class.perform_in(3600, 1)).not_to eq(nil)
         expect(described_class.perform_async(1)).to eq(nil)
       end
     end

data/lib/sidekiq_unique_jobs.rb

@@ -11,6 +11,7 @@ require 'sidekiq_unique_jobs/logging'
 require 'sidekiq_unique_jobs/sidekiq_worker_methods'
 require 'sidekiq_unique_jobs/connection'
 require 'sidekiq_unique_jobs/exceptions'
+require 'sidekiq_unique_jobs/job'
 require 'sidekiq_unique_jobs/util'
 require 'sidekiq_unique_jobs/digests'
 require 'sidekiq_unique_jobs/cli'
@@ -42,8 +43,7 @@ module SidekiqUniqueJobs
 
   module_function
 
-  Concurrent::MutableStruct.new(
-    'Config',
+  Config = Concurrent::MutableStruct.new(
     :default_lock_timeout,
     :enabled,
     :unique_prefix,
@@ -53,7 +53,7 @@ module SidekiqUniqueJobs
   # The current configuration (See: {.configure} on how to configure)
   def config
     # Arguments here need to match the definition of the new class (see above)
-    @config ||= Concurrent::MutableStruct::Config.new(
+    @config ||= Config.new(
       0,
       true,
       'uniquejobs',

data/lib/sidekiq_unique_jobs/client/middleware.rb

@@ -38,13 +38,17 @@ module SidekiqUniqueJobs
       end
 
       def locked?
-        locked = lock.lock
-        warn_about_duplicate unless locked
-        locked
+        SidekiqUniqueJobs::Job.add_uniqueness(item)
+        Sidekiq::Logging.with_context(logging_context(self.class, item)) do
+          locked = lock.lock
+          warn_about_duplicate unless locked
+          locked
+        end
       end
 
       def warn_about_duplicate
         return unless log_duplicate_payload?
+
         log_warn "payload is not unique #{item}"
       end
     end

data/lib/sidekiq_unique_jobs/constants.rb

@@ -19,4 +19,6 @@ module SidekiqUniqueJobs
   ON_CONFLICT_KEY           ||= 'on_conflict'
   UNIQUE_ON_ALL_QUEUES_KEY  ||= 'unique_on_all_queues' # TODO: Remove in v6.1
   UNIQUE_PREFIX_KEY         ||= 'unique_prefix'
+  RETRY_SET                 ||= 'retry'
+  SCHEDULE_SET              ||= 'schedule'
 end

data/lib/sidekiq_unique_jobs/job.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Utility class to append uniqueness to the sidekiq job hash
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  module Job
+    extend self # rubocop:disable Style/ModuleFunction
+
+    # Adds timeout, expiration, unique_args, unique_prefix and unique_digest to the sidekiq job hash
+    # @return [void] nothing returned here matters
+    def add_uniqueness(item)
+      add_timeout_and_expiration(item)
+      add_unique_args_and_digest(item)
+    end
+
+    private
+
+    def add_timeout_and_expiration(item)
+      calculator                = SidekiqUniqueJobs::Timeout::Calculator.new(item)
+      item[LOCK_TIMEOUT_KEY]    = calculator.lock_timeout
+      item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
+    end
+
+    def add_unique_args_and_digest(item)
+      SidekiqUniqueJobs::UniqueArgs.digest(item)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -13,9 +13,10 @@ module SidekiqUniqueJobs
       # @param [Proc] callback the callback to use after unlock
       # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
       def initialize(item, callback, redis_pool = nil)
-        @item       = prepare_item(item)
+        @item       = item
         @callback   = callback
         @redis_pool = redis_pool
+        add_uniqueness_when_missing # Used to ease testing
       end
 
       # Handles locking of sidekiq jobs.
@@ -65,6 +66,14 @@ module SidekiqUniqueJobs
 
       private
 
+      def add_uniqueness_when_missing
+        return if item.key?(UNIQUE_DIGEST_KEY)
+
+        # The below should only be done to ease testing
+        # in production this will be done by the middleware
+        SidekiqUniqueJobs::Job.add_uniqueness(item)
+      end
+
       def call_strategy
         @attempt += 1
         strategy.call { lock if replace? }
@@ -99,27 +108,14 @@ module SidekiqUniqueJobs
       def with_cleanup
         yield
       rescue Sidekiq::Shutdown
-        notify_about_manual_unlock
+        log_info('Sidekiq is shutting down, the job `should` be put back on the queue. Keeping the lock!')
         raise
       else
         unlock_with_callback
       end
 
-      def prepare_item(item)
-        calculator = SidekiqUniqueJobs::Timeout::Calculator.new(item)
-        item[LOCK_TIMEOUT_KEY]    = calculator.lock_timeout
-        item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
-        SidekiqUniqueJobs::UniqueArgs.digest(item)
-        item
-      end
-
-      def notify_about_manual_unlock
-        log_fatal("the unique_key: #{item[UNIQUE_DIGEST_KEY]} needs to be unlocked manually")
-        false
-      end
-
       def unlock_with_callback
-        return notify_about_manual_unlock unless unlock
+        return log_warn('might need to be unlocked manually') unless unlock
 
         callback_safely
         item[JID_KEY]
@@ -128,7 +124,7 @@ module SidekiqUniqueJobs
       def callback_safely
         callback&.call
       rescue StandardError
-        log_warn("The unique_key: #{item[UNIQUE_DIGEST_KEY]} has been unlocked but the #after_unlock callback failed!")
+        log_warn('unlocked successfully but the #after_unlock callback failed!')
         raise
       end
 

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -17,6 +17,7 @@ module SidekiqUniqueJobs
       # @yield to the worker class perform method
       def execute
         return unless locked?
+
         unlock
 
         runtime_lock.execute { yield }

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -14,6 +14,7 @@ module SidekiqUniqueJobs
       # @yield to the worker class perform method
       def execute
         return unless locked?
+
         with_cleanup { yield }
       end
     end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
   class Lock
     # Locks jobs until {#execute} starts
     # - Locks on perform_in or perform_async
-    # - Unlocks after yielding to the worker's perform method
+    # - Unlocks before yielding to the worker's perform method
     #
     # @author Mikael Henriksson <mikael@zoolutions.se>
     class UntilExecuting < BaseLock

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -21,6 +21,7 @@ module SidekiqUniqueJobs
       # @yield to the worker class perform method
       def execute
         return unless locked?
+
         yield
         # this lock does not handle after_unlock since we don't know when that would happen
       end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -33,6 +33,7 @@ module SidekiqUniqueJobs
       # @yield to the worker class perform method
       def execute
         return strategy.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
+
         with_cleanup { yield }
       end
 

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -37,6 +37,7 @@ module SidekiqUniqueJobs
     # Deletes the lock unless it has an expiration set
     def delete
       return if expiration
+
       delete!
     end
 
@@ -72,6 +73,7 @@ module SidekiqUniqueJobs
     def unlock(token = nil)
       token ||= jid
       return false unless locked?(token)
+
       unlock!(token)
     end
 

data/lib/sidekiq_unique_jobs/logging.rb

@@ -49,5 +49,10 @@ module SidekiqUniqueJobs
     def log_fatal(message_or_exception = nil, &block)
       logger.fatal(message_or_exception, &block)
     end
+
+    def logging_context(middleware_class, job_hash)
+      digest = job_hash['unique_digest']
+      "#{middleware_class} #{"DIG-#{digest}" if digest}"
+    end
   end
 end

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -19,13 +19,18 @@ module SidekiqUniqueJobs
       # @yield to retry the lock after deleting the old one
       def call(&block)
         return unless delete_job_by_digest
+
         delete_lock
         block&.call
       end
 
       # Delete the job from either schedule, retry or the queue
       def delete_job_by_digest
-        Scripts.call(:delete_job_by_digest, nil, keys: [queue, unique_digest])
+        Scripts.call(
+          :delete_job_by_digest,
+          nil,
+          keys: ["#{QUEUE_KEY}:#{queue}", SCHEDULE_SET, RETRY_SET], argv: [unique_digest],
+        )
       end
 
       # Delete the keys belonging to the job

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -11,12 +11,12 @@ module SidekiqUniqueJobs
   module OptionsWithFallback
     LOCKS = {
       until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
-      until_executed:            SidekiqUniqueJobs::Lock::UntilExecuted,
-      until_executing:           SidekiqUniqueJobs::Lock::UntilExecuting,
-      until_expired:             SidekiqUniqueJobs::Lock::UntilExpired,
-      until_timeout:             SidekiqUniqueJobs::Lock::UntilExpired,
-      while_executing:           SidekiqUniqueJobs::Lock::WhileExecuting,
-      while_executing_reject:    SidekiqUniqueJobs::Lock::WhileExecutingReject,
+      until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
+      until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
+      until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
+      until_timeout: SidekiqUniqueJobs::Lock::UntilExpired,
+      while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
+      while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
     }.freeze
 
     def self.included(base)

data/lib/sidekiq_unique_jobs/server/middleware.rb

@@ -6,6 +6,7 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@zoolutions.se>
     class Middleware
+      include Logging
       include OptionsWithFallback
 
       # Runs the server middleware
@@ -21,13 +22,18 @@ module SidekiqUniqueJobs
         @queue        = queue
         return yield if unique_disabled?
 
-        lock.execute do
-          yield
+        SidekiqUniqueJobs::Job.add_uniqueness(item)
+        Sidekiq::Logging.with_context(logging_context(self.class, item)) do
+          lock.execute do
+            yield
+          end
         end
       end
 
-      protected
+      private
 
+      # The sidekiq job hash
+      # @return [Hash] the Sidekiq job hash
       attr_reader :item
     end
   end

data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb

@@ -15,6 +15,7 @@ module SidekiqUniqueJobs
     # @return [Hash] of the worker class sidekiq options
     def worker_options
       return {} unless sidekiq_worker_class?
+
       worker_class.get_sidekiq_options.stringify_keys
     end
 
@@ -43,6 +44,7 @@ module SidekiqUniqueJobs
     # @return [Sidekiq::Worker]
     def worker_class_constantize(klazz = @worker_class)
       return klazz unless klazz.is_a?(String)
+
       Object.const_get(klazz)
     rescue NameError => ex
       case ex.message

data/lib/sidekiq_unique_jobs/timeout/calculator.rb

@@ -25,6 +25,7 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of seconds until job is scheduled
       def time_until_scheduled
         return 0 unless scheduled_at
+
         scheduled_at.to_i - Time.now.utc.to_i
       end
 
@@ -39,7 +40,7 @@ module SidekiqUniqueJobs
         @lock_expiration ||= begin
           expiration = item[LOCK_EXPIRATION_KEY]
           expiration ||= worker_options[LOCK_EXPIRATION_KEY]
-          expiration && expiration + time_until_scheduled
+          expiration && expiration.to_i + time_until_scheduled
         end
       end
 

data/lib/sidekiq_unique_jobs/unique_args.rb

@@ -70,6 +70,7 @@ module SidekiqUniqueJobs
     # @return [Array] the arguments filters by the {#filtered_args} method if {#unique_args_enabled?}
     def unique_args(args)
       return filtered_args(args) if unique_args_enabled?
+
       args
     end
 
@@ -99,6 +100,7 @@ module SidekiqUniqueJobs
     # @return [Array] args unfiltered when neither of the above
     def filtered_args(args)
       return args if args.empty?
+
       json_args = Normalizer.jsonify(args)
 
       case unique_args_method

data/lib/sidekiq_unique_jobs/util.rb

@@ -20,6 +20,7 @@ module SidekiqUniqueJobs
     # @return [Array<String>] an array with active unique keys
     def keys(pattern = SCAN_PATTERN, count = DEFAULT_COUNT)
       return redis(&:keys) if pattern.nil?
+
       redis { |conn| conn.scan_each(match: prefix(pattern), count: count).to_a }
     end
 
@@ -44,6 +45,7 @@ module SidekiqUniqueJobs
     # @return [Integer] the number of keys deleted
     def del(pattern = SCAN_PATTERN, count = 0)
       raise ArgumentError, 'Please provide a number of keys to delete greater than zero' if count.zero?
+
       pattern = suffix(pattern)
 
       log_debug { "Deleting keys by: #{pattern}" }
@@ -84,11 +86,13 @@ module SidekiqUniqueJobs
     def prefix(key)
       return key if unique_prefix.nil?
       return key if key.start_with?("#{unique_prefix}:")
+
       "#{unique_prefix}:#{key}"
     end
 
     def suffix(key)
       return "#{key}*" unless key.end_with?(':*')
+
       key
     end
 

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  VERSION = '6.0.6'
+  VERSION = '6.0.7'
 end

data/lib/sidekiq_unique_jobs/web/helpers.rb

@@ -19,6 +19,7 @@ module SidekiqUniqueJobs
 
         params.merge(options).map do |key, value|
           next unless SAFE_CPARAMS.include?(key)
+
           "#{key}=#{CGI.escape(value.to_s)}"
         end.compact.join('&')
       end

data/redis/delete_job_by_digest.lua

@@ -1,5 +1,7 @@
-local queue         = "queue:" .. KEYS[1]
-local unique_digest = KEYS[2]
+local queue         = KEYS[1]
+local schedule_set  = KEYS[2]
+local retry_set     = KEYS[3]
+local unique_digest = ARGV[1]
 
 local function delete_from_sorted_set(name, digest)
   local per   = 50
@@ -49,10 +51,10 @@ if result then
   return result
 end
 
-result = delete_from_sorted_set('schedule', unique_digest)
+result = delete_from_sorted_set(schedule_set, unique_digest)
 if result then
   return result
 end
 
-result = delete_from_sorted_set('retry', unique_digest)
+result = delete_from_sorted_set(retry_set, unique_digest)
 return result

data/redis/unlock.lua

@@ -10,7 +10,6 @@ local expiration = tonumber(ARGV[2])
 
 redis.call('HDEL', grabbed_key, token)
 redis.call('SREM', unique_keys, unique_digest)
-local available_count = redis.call('LPUSH', available_key, token)
 
 if expiration then
   redis.log(redis.LOG_DEBUG, "signal_locks.lua - expiring stale locks")
@@ -29,5 +28,5 @@ else
   redis.call('DEL', unique_digest)  -- TODO: Legacy support (Remove in v6.1)
 end
 
-return available_count
+return redis.call('LPUSH', available_key, token)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 6.0.6
+  version: 6.0.7
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-09 00:00:00.000000000 Z
+date: 2018-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -273,6 +273,7 @@ files:
 - lib/sidekiq_unique_jobs/core_ext.rb
 - lib/sidekiq_unique_jobs/digests.rb
 - lib/sidekiq_unique_jobs/exceptions.rb
+- lib/sidekiq_unique_jobs/job.rb
 - lib/sidekiq_unique_jobs/lock/base_lock.rb
 - lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb
 - lib/sidekiq_unique_jobs/lock/until_executed.rb