sidekiq-unique-jobs 6.0.10 → 6.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fbbb9d542c2f503e41c0c1870ce0b3c924889060218f5e27328f266239476197
-  data.tar.gz: 2d11eb5bd79e978f2cd03db40af5f443c7431286028cca3b28249a48764c9600
+  metadata.gz: 4794cab4e8c46ab8a1c2f8d3f1f4a0e40eee2f5c0a78c1750816a4f08298fb3e
+  data.tar.gz: 5542da14289cd0287001c997b439f77445c9d7b7c13cc8307801a55012217a7c
 SHA512:
-  metadata.gz: 3245b73d1d2c432f8bbeb35c1b3b5eeaba7ed82283f0ff31e769ec887e48ef4dba1cbcf91e6fd59d28710a4ae801ac6914570d96c360713c61adb12391373c6c
-  data.tar.gz: 940daa0baa646433ba4c83a310cd17b84aad1de814aa8714354d85cd69aa184fbf76d5c17dee3e30e0c2b4ec0504027473dc8c2a83474e8bd9b1c37ef8b198f8
+  metadata.gz: ee3cddb22fa61d68e5b209fbd129c5e8c1c88c3cb02feae0f1d0b869d3722ab9f8e5d08cf0a2d0bbcbb11be2205ee35e5f9202c24eb56cae360e79e4bacd54c9
+  data.tar.gz: e0c980ae65ac375cf0de4595b61ca4137c52ba0335d947560d7d7550f71db8873a4dca228e18f3319c3cdd551b7377a262133c1d7b66ad8daf9a8143b3fdd4f2

data/.travis.yml

@@ -38,6 +38,10 @@ matrix:
     - rvm: 2.5.3
       gemfile: gemfiles/sidekiq_develop.gemfile
       env: COV=true
+    - rvm: 2.6.0
+      gemfile: gemfiles/sidekiq_6.0.gemfile
+    - rvm: 2.5.3
+      gemfile: gemfiles/sidekiq_6.0.gemfile
 
 gemfile:
   - gemfiles/sidekiq_develop.gemfile

data/Appraisals

@@ -27,3 +27,7 @@ end
 appraise 'sidekiq-5.2' do
   gem 'sidekiq', '~> 5.2.0'
 end
+
+appraise 'sidekiq-6.0' do
+  gem 'sidekiq', git: 'https://github.com/mperham/sidekiq.git', branch: '6-0'
+end

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Change Log
 
+## [v6.0.10](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v6.0.10) (2019-02-23)
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v6.0.9...v6.0.10)
+
+**Closed issues:**
+
+- Unsure of sane defaults [\#372](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/372)
+
+**Merged pull requests:**
+
+- Log job silently complete [\#371](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/371) ([tadejm](https://github.com/tadejm))
+
 ## [v6.0.9](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v6.0.9) (2019-02-11)
 [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v6.0.8...v6.0.9)
 

data/Gemfile

@@ -7,6 +7,7 @@ gem "appraisal", "~> 2.2.0"
 gem "rspec-eventually", require: false
 gem "rspec-its",        require: false
 gem "rspec-retry",      require: false
+gem "sidekiq", git: "https://github.com/mperham/sidekiq.git", branch: "6-0"
 
 platforms :mri_25 do
   gem "benchmark-ips"

data/README.md

@@ -94,15 +94,13 @@ See [Locking & Unlocking](https://github.com/mhenrixon/sidekiq-unique-jobs/wiki/
 
 ### Lock Expiration
 
-This is probably not the configuration option you want...
+Lock expiration is used for two things. For the `UntilExpired` job releases the lock upon expiry. This is done from the client.
 
-Since the client and the server are disconnected and not running inside the same process, setting a lock expiration is probably not what you want. Any keys that are used by this gem WILL be removed at the time of the expiration. For jobs that are scheduled in the future the key will expire when that job is scheduled + whatever expiration you have set.
-
-In previous versions there was a default expiration of 30 minutes which didn't work for a lot of long running jobs. Since version 6 there will be no expiration of any jobs from the default configuration. Please don't use `lock_expiration` unless you really know what you are doing.
+Since v6.0.11 the other locks will expire after the server is done processing.
 
 ```ruby
 sidekiq_options lock_expiration: nil # default - don't expire keys
-sidekiq_options lock_expiration: 20.days # expire this lock in 20 days
+sidekiq_options lock_expiration: 20.days.to_i # expire this lock in 20 days
 ```
 
 ### Lock Timeout
@@ -363,7 +361,7 @@ end
 
 ### Cleanup Dead Locks
 
-For sidekiq versions before 5.1 a `sidekiq_retries_exhausted` block is required per worker class.
+For sidekiq versions before 5.1 a `sidekiq_retries_exhausted` block is required per worker class. This is deprecated in Sidekiq 6.0
 
 ```ruby
 class MyWorker

data/lib/sidekiq_unique_jobs/client/middleware.rb

@@ -39,7 +39,7 @@ module SidekiqUniqueJobs
 
       def locked?
         SidekiqUniqueJobs::Job.add_uniqueness(item)
-        Sidekiq::Logging.with_context(logging_context(self.class, item)) do
+        SidekiqUniqueJobs.with_context(logging_context(self.class, item)) do
           locked = lock.lock
           warn_about_duplicate unless locked
           locked

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -4,6 +4,7 @@ module SidekiqUniqueJobs
   # Lock manager class that handles all the various locks
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>
+  # rubocop:disable ClassLength
   class Locksmith
     include SidekiqUniqueJobs::Connection
 
@@ -13,35 +14,23 @@ module SidekiqUniqueJobs
     # @option item [String] :unique_digest the unique digest (See: {UniqueArgs#unique_digest})
     # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
     def initialize(item, redis_pool = nil)
-      @concurrency   = 1 # removed in a0cff5bc42edbe7190d6ede7e7f845074d2d7af6
-      @expiration    = item[LOCK_EXPIRATION_KEY]
+      # @concurrency   = 1 # removed in a0cff5bc42edbe7190d6ede7e7f845074d2d7af6
+      @ttl           = item[LOCK_EXPIRATION_KEY]
       @jid           = item[JID_KEY]
       @unique_digest = item[UNIQUE_DIGEST_KEY]
+      @lock_type     = item[LOCK_KEY]
+      @lock_type   &&= @lock_type.to_sym
       @redis_pool    = redis_pool
     end
 
-    # Checks if the exists key is created in redis
-    # @return [true, false]
-    def exists?
-      redis(redis_pool) { |conn| conn.exists(exists_key) }
-    end
-
-    # The number of available resourced for this lock
-    # @return [Integer] the number of available resources
-    def available_count
-      return concurrency unless exists?
-
-      redis(redis_pool) { |conn| conn.llen(available_key) }
-    end
-
-    # Deletes the lock unless it has an expiration set
+    # Deletes the lock unless it has a ttl set
     def delete
-      return if expiration
+      return if ttl
 
       delete!
     end
 
-    # Deletes the lock regardless of if it has an expiration set
+    # Deletes the lock regardless of if it has a ttl set
     def delete!
       Scripts.call(
         :delete,
@@ -58,7 +47,7 @@ module SidekiqUniqueJobs
     def lock(timeout = nil, &block)
       Scripts.call(:lock, redis_pool,
                    keys: [exists_key, grabbed_key, available_key, UNIQUE_SET, unique_digest],
-                   argv: [jid, expiration])
+                   argv: [jid, ttl, lock_type])
 
       grab_token(timeout) do |token|
         touch_grabbed_token(token)
@@ -87,7 +76,7 @@ module SidekiqUniqueJobs
         :unlock,
         redis_pool,
         keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
-        argv: [token, expiration],
+        argv: [token, ttl, lock_type],
       )
     end
 
@@ -97,12 +86,19 @@ module SidekiqUniqueJobs
     # @return [true, false]
     def locked?(token = nil)
       token ||= jid
+      Scripts.call(
+        :convert_legacy_lock,
+        redis_pool,
+        keys: [grabbed_key, unique_digest],
+        argv: [token],
+      )
+
       redis(redis_pool) { |conn| conn.hexists(grabbed_key, token) }
     end
 
     private
 
-    attr_reader :concurrency, :unique_digest, :expiration, :jid, :redis_pool
+    attr_reader :unique_digest, :ttl, :jid, :redis_pool, :lock_type
 
     def grab_token(timeout = nil)
       redis(redis_pool) do |conn|
@@ -118,7 +114,10 @@ module SidekiqUniqueJobs
     end
 
     def touch_grabbed_token(token)
-      redis(redis_pool) { |conn| conn.hset(grabbed_key, token, current_time.to_f) }
+      redis(redis_pool) do |conn|
+        conn.hset(grabbed_key, token, current_time.to_f)
+        conn.expire(grabbed_key, ttl) if ttl && lock_type == :until_expired
+      end
     end
 
     def return_token_or_block_value(token)
@@ -161,4 +160,5 @@ module SidekiqUniqueJobs
       redis(&:time)
     end
   end
+  # rubocop:enable ClassLength
 end

data/lib/sidekiq_unique_jobs/logging.rb

@@ -52,7 +52,11 @@ module SidekiqUniqueJobs
 
     def logging_context(middleware_class, job_hash)
       digest = job_hash["unique_digest"]
-      "#{middleware_class} #{"DIG-#{digest}" if digest}"
+      if defined?(Sidekiq::Logging)
+        "#{middleware_class} #{"DIG-#{digest}" if digest}"
+      else
+        { middleware: middleware_class, unique_digest: digest }
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/server/middleware.rb

@@ -23,7 +23,7 @@ module SidekiqUniqueJobs
         return yield if unique_disabled?
 
         SidekiqUniqueJobs::Job.add_uniqueness(item)
-        Sidekiq::Logging.with_context(logging_context(self.class, item)) do
+        SidekiqUniqueJobs.with_context(logging_context(self.class, item)) do
           lock.execute do
             yield
           end

data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb

@@ -33,6 +33,15 @@ module SidekiqUniqueJobs
     config.logger
   end
 
+  # :reek:ManualDispatch
+  def with_context(context, &block)
+    if logger.respond_to?(:with_context)
+      logger.with_context(context, &block)
+    elsif defined?(Sidekiq::Logging)
+      Sidekiq::Logging.with_context(context, &block)
+    end
+  end
+
   # Set a new logger
   # @param [Logger] other a new logger
   def logger=(other)

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  VERSION = "6.0.10"
+  VERSION = "6.0.11"
 end

data/redis/convert_legacy_lock.lua

@@ -0,0 +1,21 @@
+local grabbed_key   = KEYS[1]
+local unique_digest = KEYS[2]
+
+local job_id = ARGV[1]
+
+local function current_time()
+  local time = redis.call('time')
+  local s = time[1]
+  local ms = time[2]
+  local number = tonumber((s .. '.' .. ms))
+
+  return number
+end
+
+local old_token  = redis.call('GET', unique_digest)
+if old_token then
+  if old_token == job_id or old_token == '2' then
+    redis.call('DEL', unique_digest)
+    redis.call('HSET', grabbed_key, job_id, current_time())
+  end
+end

data/redis/lock.lua

@@ -6,8 +6,9 @@ local available_key = KEYS[3]
 local unique_keys   = KEYS[4]
 local unique_digest = KEYS[5]
 
-local job_id        = ARGV[1]
-local expiration    = tonumber(ARGV[2])
+local job_id = ARGV[1]
+local ttl    = tonumber(ARGV[2])
+local lock   = ARGV[3]
 
 local function current_time()
   local time = redis.call('time')
@@ -18,11 +19,8 @@ local function current_time()
   return number
 end
 
--- redis.log(redis.LOG_DEBUG, "create.lua - investigate possibility of locking jid: " .. job_id)
-
-local stored_token  = redis.call('GET', exists_key)
+local stored_token = redis.call('GET', exists_key)
 if stored_token and stored_token ~= job_id then
-  -- redis.log(redis.LOG_DEBUG, "create.lua - jid: " .. job_id .. " - returning existing jid: " .. stored_token)
   return stored_token
 end
 
@@ -49,10 +47,15 @@ redis.call('DEL', grabbed_key)
 redis.call('DEL', available_key)
 redis.call('RPUSH', available_key, job_id)
 
-if expiration then
-  redis.call('EXPIRE', available_key, expiration)
-  redis.call('EXPIRE', exists_key, expiration)
-  redis.call('EXPIRE', grabbed_key, expiration)
+-- The client should only set ttl for until_expired
+-- The server should set ttl for all other jobs
+if lock == "until_expired" and ttl then
+  -- We can't keep the key here because it will otherwise never be deleted
+  redis.call('SREM', unique_keys, unique_digest)
+
+  redis.call('EXPIRE', available_key, ttl)
+  redis.call('EXPIRE', exists_key, ttl)
+  redis.call('EXPIRE', unique_digest, ttl)
 end
 
 return job_id

data/redis/unlock.lua

@@ -5,19 +5,19 @@ local version_key   = KEYS[4]
 local unique_keys   = KEYS[5]
 local unique_digest = KEYS[6] -- TODO: Legacy support (Remove in v6.1)
 
-local token      = ARGV[1]
-local expiration = tonumber(ARGV[2])
+local token = ARGV[1]
+local ttl   = tonumber(ARGV[2])
+local lock  = ARGV[3]
 
-redis.call('HDEL', grabbed_key, token)
 redis.call('SREM', unique_keys, unique_digest)
 
-if expiration then
-  redis.log(redis.LOG_DEBUG, "signal_locks.lua - expiring stale locks")
+if ttl then
   redis.call('SREM', unique_keys, unique_digest)
-  redis.call('EXPIRE', exists_key, expiration)
-  redis.call('EXPIRE', available_key, expiration)
-  redis.call('EXPIRE', version_key, expiration)   -- TODO: Legacy support (Remove in v6.1)
-  redis.call('EXPIRE', unique_digest, expiration) -- TODO: Legacy support (Remove in v6.1)
+  redis.call('EXPIRE', exists_key, ttl)
+  redis.call('EXPIRE', grabbed_key, ttl)
+  redis.call('EXPIRE', available_key, ttl)
+  redis.call('EXPIRE', version_key, ttl)   -- TODO: Legacy support (Remove in v6.1)
+  redis.call('EXPIRE', unique_digest, ttl) -- TODO: Legacy support (Remove in v6.1)
 else
   redis.call('DEL', exists_key)
   redis.call('SREM', unique_keys, unique_digest)
@@ -28,6 +28,7 @@ else
   redis.call('DEL', unique_digest)  -- TODO: Legacy support (Remove in v6.1)
 end
 
+redis.call('HDEL', grabbed_key, token)
 local count = redis.call('LPUSH', available_key, token)
 redis.call('EXPIRE', available_key, 5)
 return count

data/sidekiq-unique-jobs.gemspec

@@ -42,7 +42,7 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ["lib"]
   spec.add_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.5"
-  spec.add_dependency "sidekiq", ">= 4.0", "< 6.0"
+  spec.add_dependency "sidekiq", ">= 4.0", "< 7.0"
   spec.add_dependency "thor", "~> 0"
 
   spec.add_development_dependency "bundler", ">= 2.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 6.0.10
+  version: 6.0.11
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-23 00:00:00.000000000 Z
+date: 2019-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -39,7 +39,7 @@ dependencies:
         version: '4.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -49,7 +49,7 @@ dependencies:
         version: '4.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -340,6 +340,7 @@ files:
 - lib/sidekiq_unique_jobs/web/views/unique_digests.erb
 - lib/tasks/changelog.rake
 - redis/acquire_lock.lua
+- redis/convert_legacy_lock.lua
 - redis/delete.lua
 - redis/delete_by_digest.lua
 - redis/delete_job_by_digest.lua