sidekiq-unique-jobs 5.0.0 → 5.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 133211c0920216c3537ec4417d74252f4804420b
-  data.tar.gz: 3b2f8878d4ea75aaacaa9e1c82876705aefad514
+  metadata.gz: 54089588f5279692be7686eca3e9b4987c9cb706
+  data.tar.gz: 2ce8320f727817419868df6ddc877d143c9318cf
 SHA512:
-  metadata.gz: aca4b5c230b92c0bd7b180e3738bb930e82309743008b2c17726012d851623f5a2da156141872980942a22c84e8dfefe0affe77a212d4a210d6bc8d072f47af3
-  data.tar.gz: cfaaa3845e5bc4012df8001bb23f01f352fe99ece7e8330020b7dbf19c3e126ca76a306584dc50a76b909d279ce701b32b51ba0ca7e9b762123ea909d2924672
+  metadata.gz: fe2abbb06c27ef94b8266d1bcc1727b1385e7d6897c35731986d90306ce218b16b7d22336facbfff95b43e294abf5dbe4b894ce4d6629d4373e272cb20285502
+  data.tar.gz: eba5e89015ec74852a4a6e0615e3793ede42dee32a22e59ccd019de3a22e14b27ea27dbb757c5a4cf6cc3eab2bf80d2a6801928bb8a4030c6c3dfb45e31df593

data/.codeclimate.yml

@@ -1,36 +1,7 @@
 ---
-engines:
-  csslint:
-    enabled: true
-  duplication:
-    enabled: true
-    config:
-      languages:
-      - ruby
-      - javascript
-      - python
-      - php
-  eslint:
-    enabled: true
-  fixme:
-    enabled: true
-  rubocop:
-    enabled: true
-ratings:
-  paths:
-  - "**.css"
-  - "**.inc"
-  - "**.js"
-  - "**.jsx"
-  - "**.module"
-  - "**.php"
-  - "**.py"
-  - "**.rb"
-exclude_paths:
-  - spec/
 engines:
   bundler-audit:
-    enabled: true
+    enabled: false
   fixme:
     enabled: true
   duplication:
@@ -47,7 +18,7 @@ ratings:
     - "**.rb"
 exclude_paths:
   - Gemfile
-  - *.gemspec
+  - "*.gemspec"
   - Appraisals
   - spec/**/*.rb
   - gemfiles/**/*

data/.editorconfig

@@ -10,5 +10,5 @@ insert_final_newline = true
 indent_style = space
 indent_size = 2
 
-[*.md]
-trim_trailing_whitespace = true
+[*.{md,slim,haml}]
+trim_trailing_whitespace = false

data/.rubocop.yml

@@ -22,7 +22,7 @@ Metrics/CyclomaticComplexity:
   Max: 7
 
 Metrics/LineLength:
-  Max: 108
+  Max: 120
 
 Metrics/MethodLength:
   Max: 13

data/.travis.yml

@@ -1,10 +1,10 @@
 sudo: false
 language: ruby
 cache: bundler
-before_install:
-  - rvm get head
-  - gem update --system
-  - gem install bundler
+# before_install:
+#   - rvm get head
+#   - gem update --system
+#   - gem install bundler
 services:
   - redis-server
 script:

data/CHANGELOG.md

@@ -1,6 +1,13 @@
+## v5.0.1
+
+- Added a command line util for cleaning out expired keys
+- Use `SidekiqUniqueJobs.logger` instead of spreading out `Sidekiq.logger` everywhere.
+
 ## v5.0.0
 
 - Only support Sidekiq >= 4
+- Removed overrides and support for older Sidekiq testing
+- Added coverage
 
 ## v4.0.18
 

data/README.md

@@ -5,15 +5,11 @@ The missing unique jobs for sidekiq
 ## Requirements
 
 
-See https://github.com/mperham/sidekiq#requirements for what is required. Starting from 3.0.13 only sidekiq 3 is supported and support for MRI 1.9 is dropped (it might work but won't be worked on)
+See https://github.com/mperham/sidekiq#requirements for what is required. Starting from 5.0.0 only sidekiq >= 4 is supported and support for MRI <=  2.1 is dropped.
 
 ### Version 4 Upgrade instructions
 
-Version 4 requires redis >= 2.6.2!! Don't upgrade to version 4 unless you are on redis >= 2.6.2.
-
-Easy path - Drop all your unique jobs before upgrading the gem!
-
-Hard path - See below... Start with a clean slate :)
+Version 5 requires redis >= 3
 
 ## Installation
 

data/lib/sidekiq-unique-jobs.rb

@@ -26,10 +26,14 @@ module SidekiqUniqueJobs
       default_queue_lock_expiration: 30 * 60,
       default_run_lock_expiration: 60,
       default_lock: :while_executing,
-      redis_test_mode: :redis # :mock
+      redis_test_mode: :redis, # :mock
     )
   end
 
+  def logger
+    Sidekiq.logger
+  end
+
   def default_lock
     config.default_lock
   end

data/lib/sidekiq/simulator.rb

@@ -3,6 +3,9 @@ require 'timeout'
 
 module Sidekiq
   class Simulator
+    extend Forwardable
+    def_delegator SidekiqUniqueJobs, :logger
+
     attr_reader :queues, :launcher
 
     def self.process_queue(queue)
@@ -67,9 +70,5 @@ module Sidekiq
         verbose: false,
         logfile: './tmp/sidekiq.log' }
     end
-
-    def logger
-      @logger ||= Sidekiq.logger
-    end
   end
 end

data/lib/sidekiq_unique_jobs/cli.rb

@@ -2,30 +2,52 @@ require 'thor'
 
 module SidekiqUniqueJobs
   class Cli < Thor
+    # def initialize(argv, stdin = STDIN, stdout = STDOUT, stderr = STDERR, kernel = Kernel)
+    #   @argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
+    # end
+
+    def self.banner(command, _namespace = nil, _subcommand = false)
+      "jobs #{@package_name} #{command.usage}"
+    end
+
     desc 'keys PATTERN', 'list all unique keys and their expiry time'
     option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
     def keys(pattern)
-      Util.keys(pattern, options[:count])
+      keys = Util.keys(pattern, options[:count])
+      say "Found #{keys.size} keys matching '#{pattern}':"
+      print_in_columns(keys.sort) if keys.any?
     end
 
     desc 'del PATTERN', 'deletes unique keys from redis by pattern'
     option :dry_run, aliases: :d, type: :boolean, desc: 'set to false to perform deletion'
     option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
     def del(pattern)
-      Util.del(pattern, options[:count], options[:dry_run])
+      deleted_count = Util.del(pattern, options[:count], options[:dry_run])
+      say "Deleted #{deleted_count} keys matching '#{pattern}'"
+    end
+
+    desc 'expire', 'removes all expired unique keys from the hash in redis'
+    def expire
+      expired = Util.expire
+      say "Removed #{expired.values.size} left overs from redis."
+      print_in_columns(expired.values)
     end
 
     desc 'console', 'drop into a console with easy access to helper methods'
     def console
-      puts "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
-      puts "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"
-      puts "Use `del '*', 1000, false to delete the first 1000 keys matching '*'"
+      say "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
+      say "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"
+      say "Use `del '*', 1000, false to delete the first 1000 keys matching '*'"
       Object.include SidekiqUniqueJobs::Util
       console_class.start
     end
 
     private
 
+    def logger
+      SidekiqUniqueJobs.logger
+    end
+
     def console_class
       require 'pry'
       Pry

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -35,25 +35,17 @@ module SidekiqUniqueJobs
         unlock_by_key(unique_key, item[JID_KEY], redis_pool)
       end
 
-      # rubocop:disable MethodLength
       def lock(scope)
         if scope.to_sym != :client
           raise ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
         end
 
-        result = Scripts.call(:acquire_lock, redis_pool,
-                              keys: [unique_key],
-                              argv: [item[JID_KEY], max_lock_time])
-        case result
-        when 1
-          logger.debug { "successfully locked #{unique_key} for #{max_lock_time} seconds" }
-          true
-        when 0
-          logger.debug { "failed to acquire lock for #{unique_key}" }
-          false
-        else
-          raise "#{__method__} returned an unexpected value (#{result})"
-        end
+        Scripts::AcquireLock.execute(
+          redis_pool,
+          unique_key,
+          item[JID_KEY],
+          max_lock_time
+        )
       end
       # rubocop:enable MethodLength
 

data/lib/sidekiq_unique_jobs/scripts.rb

@@ -1,9 +1,15 @@
 require 'pathname'
 require 'digest/sha1'
 require 'concurrent/map'
+require 'sidekiq_unique_jobs/scripts/acquire_lock'
+require 'sidekiq_unique_jobs/scripts/release_lock'
 
 module SidekiqUniqueJobs
-  ScriptError = Class.new(StandardError)
+  ScriptError         = Class.new(StandardError)
+  UniqueKeyMissing    = Class.new(ArgumentError)
+  JidMissing          = Class.new(ArgumentError)
+  MaxLockTimeMissing  = Class.new(ArgumentError)
+  UnexpectedValue     = Class.new(StandardError)
 
   module Scripts
     LUA_PATHNAME ||= Pathname.new(__FILE__).dirname.join('../../redis').freeze
@@ -14,11 +20,7 @@ module SidekiqUniqueJobs
     module_function
 
     extend SingleForwardable
-    def_delegator :SidekiqUniqueJobs, :connection
-
-    def logger
-      Sidekiq.logger
-    end
+    def_delegators :SidekiqUniqueJobs, :connection, :logger
 
     def call(file_name, redis_pool, options = {}) # rubocop:disable MethodLength
       connection(redis_pool) do |redis|

data/lib/sidekiq_unique_jobs/scripts/acquire_lock.rb

@@ -0,0 +1,45 @@
+module SidekiqUniqueJobs
+  module Scripts
+    class AcquireLock
+      extend Forwardable
+      def_delegator SidekiqUniqueJobs, :logger
+
+      def self.execute(redis_pool, unique_key, jid, max_lock_time)
+        new(redis_pool, unique_key, jid, max_lock_time).execute
+      end
+
+      attr_reader :redis_pool, :unique_key, :jid, :max_lock_time
+
+      def initialize(_redis_pool, unique_key, jid, max_lock_time)
+        raise UniqueKeyMissing, 'unique_key is required' if unique_key.nil?
+        raise JidMissing, 'jid is required' if jid.nil?
+        raise MaxLockTimeMissing, 'max_lock_time is required' if max_lock_time.nil?
+
+        @unique_key    = unique_key
+        @jid           = jid
+        @max_lock_time = max_lock_time
+      end
+
+      def execute
+        result = Scripts.call(:acquire_lock, redis_pool,
+                              keys: [unique_key],
+                              argv: [jid, max_lock_time])
+
+        handle_result(result)
+      end
+
+      def handle_result(result)
+        case result
+        when 1
+          logger.debug { "successfully acquired lock #{unique_key} for #{max_lock_time} seconds" }
+          true
+        when 0
+          logger.debug { "failed to acquire lock for #{unique_key}" }
+          false
+        else
+          raise UnexpectedValue, "failed to acquire lock : unexpected return value (#{result})"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/scripts/release_lock.rb

@@ -0,0 +1,47 @@
+module SidekiqUniqueJobs
+  module Scripts
+    class ReleaseLock
+      extend Forwardable
+      def_delegator SidekiqUniqueJobs, :logger
+
+      def self.execute(redis_pool, unique_key, jid)
+        new(redis_pool, unique_key, jid).execute
+      end
+
+      attr_reader :redis_pool, :unique_key, :jid
+
+      def initialize(redis_pool, unique_key, jid)
+        raise UniqueKeyMissing, 'unique_key is required' if unique_key.nil?
+        raise JidMissing, 'jid is required' if jid.nil?
+
+        @redis_pool    = redis_pool
+        @unique_key    = unique_key
+        @jid           = jid
+      end
+
+      def execute
+        result = Scripts.call(:release_lock, redis_pool,
+                              keys: [unique_key],
+                              argv: [jid])
+
+        handle_result(result)
+      end
+
+      def handle_result(result)
+        case result
+        when 1
+          logger.debug { "successfully unlocked #{unique_key}" }
+          true
+        when 0
+          logger.debug { "expiring lock #{unique_key} is not owned by #{jid}" }
+          false
+        when -1
+          logger.debug { "#{unique_key} is not a known key" }
+          false
+        else
+          raise UnexpectedValue, "failed to release lock : unexpected return value (#{result})"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/testing.rb

@@ -25,7 +25,7 @@ module SidekiqUniqueJobs
 
       module Testing
         def call_ext(file_name, redis_pool, options = {})
-          if SidekiqUniqueJobs.config.redis_test_mode == :mock
+          if SidekiqUniqueJobs.mocked?
             SidekiqUniqueJobs::ScriptMock.call(file_name, redis_pool, options)
           else
             call_orig(file_name, redis_pool, options)

data/lib/sidekiq_unique_jobs/unique_args.rb

@@ -19,10 +19,10 @@ module SidekiqUniqueJobs
     def initialize(job)
       Sidekiq::Logging.with_context(CLASS_NAME) do
         @item = job
-        @worker_class ||= worker_class_constantize(@item[CLASS_KEY])
+        @worker_class            ||= worker_class_constantize(@item[CLASS_KEY])
         @item[UNIQUE_PREFIX_KEY] ||= unique_prefix
-        @item[UNIQUE_ARGS_KEY] = unique_args(@item[ARGS_KEY]) # SIC! Calculate unique_args unconditionally
-        @item[UNIQUE_DIGEST_KEY] = unique_digest
+        @item[UNIQUE_ARGS_KEY]     = unique_args(@item[ARGS_KEY])
+        @item[UNIQUE_DIGEST_KEY]   = unique_digest
       end
     end
 

data/lib/sidekiq_unique_jobs/unlockable.rb

@@ -7,26 +7,12 @@ module SidekiqUniqueJobs
     end
 
     def unlock_by_key(unique_key, jid, redis_pool = nil)
-      result = Scripts.call(:release_lock, redis_pool, keys: [unique_key], argv: [jid])
-      after_unlock(result, __method__, unique_key, jid)
+      return false unless Scripts::ReleaseLock.execute(redis_pool, unique_key, jid)
+      after_unlock(jid)
     end
 
-    def after_unlock(result, calling_method, unique_key, jid) # rubocop:disable Metrics/MethodLength
+    def after_unlock(jid)
       ensure_job_id_removed(jid)
-
-      case result
-      when 1
-        logger.debug { "successfully unlocked #{unique_key}" }
-        true
-      when 0
-        logger.debug { "expiring lock #{unique_key} is not owned by #{jid}" }
-        false
-      when -1
-        logger.debug { "#{unique_key} is not a known key" }
-        false
-      else
-        raise "#{calling_method} returned an unexpected value (#{result})"
-      end
     end
 
     def ensure_job_id_removed(jid)
@@ -34,7 +20,7 @@ module SidekiqUniqueJobs
     end
 
     def logger
-      Sidekiq.logger
+      SidekiqUniqueJobs.logger
     end
   end
 end

data/lib/sidekiq_unique_jobs/util.rb

@@ -18,7 +18,7 @@ module SidekiqUniqueJobs
     end
 
     def del(pattern = SCAN_PATTERN, count = 0, dry_run = true)
-      raise 'Please provide a number of keys to delete greater than zero' if count.zero?
+      raise ArgumentError, 'Please provide a number of keys to delete greater than zero' if count.zero?
       logger.debug { "Deleting keys by: #{pattern}" }
       keys, time = timed { keys(pattern, count) }
       logger.debug { "#{keys.size} matching keys found in #{time} sec." }
@@ -32,6 +32,24 @@ module SidekiqUniqueJobs
       keys.size
     end
 
+    def unique_hash
+      connection do |conn|
+        conn.hgetall(SidekiqUniqueJobs::HASH_KEY)
+      end
+    end
+
+    def expire
+      removed_keys = {}
+      connection do |conn|
+        conn.hgetall(SidekiqUniqueJobs::HASH_KEY).each do |jid, unique_key|
+          next if conn.get(unique_key)
+          conn.hdel(SidekiqUniqueJobs::HASH_KEY, jid)
+          removed_keys[jid] = unique_key
+        end
+      end
+      removed_keys
+    end
+
     def keys_by_scan(pattern, count)
       connection { |conn| conn.scan_each(match: prefix(pattern), count: count).to_a }
     end
@@ -72,6 +90,7 @@ module SidekiqUniqueJobs
 
     def prefix(key)
       return key if unique_prefix.nil?
+      return key if key.start_with?("#{unique_prefix}:")
       "#{unique_prefix}:#{key}"
     end
 
@@ -92,7 +111,7 @@ module SidekiqUniqueJobs
     end
 
     def logger
-      Sidekiq.logger
+      SidekiqUniqueJobs.logger
     end
   end
 end