sidekiq-unique-jobs 6.0.0.rc7 → 6.0.0.rc8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c90e3f7dec1b958a7516b7fe9aa703622abba4e7713c1e18d3022c6f853822b2
-  data.tar.gz: b3597746d75ce16841f2dc91bd5ba1377850f1341e4bece3478f803f61840979
+  metadata.gz: e9d567f37cb10954057a83832bfcaba08b50ee5306c0acd5072f00c2cecbc9be
+  data.tar.gz: a76383d74c088f76fb41a56166b5fec7d4cbec89ec4fe3a5eb82f831b18b43e2
 SHA512:
-  metadata.gz: fd3fb117d369c9e5ce97d5ea175813e50f76ae59c41cc6aaf91accc9bc613a0cefffd8d20420d854a7378125b0108ddef603011bc0fede46fa0addd7785ef057
-  data.tar.gz: 9d416d37175611a6de8d8dbe9e416218f5439a9760577d7b337df7b0b8f21da2d35c8a0a4ba4846d8490fe26fa467f21f258c5645701cf81fb9e87d276243434
+  metadata.gz: 41a3ebbddb28fd0e3b60493c481b454a8f5562d4e0fd69f20ee19c72cf68d90ab0e8c7492251fc0b19d6b3fb2693898b54c2a04198651150c7f4687d7c71b7cd
+  data.tar.gz: e62237661c82579a110af614781bbfaf4d0a7431c484d793c6032428e9158b55df7095df7848044b412edd387478fbce808432242736b16aaab8efd6ab8c0c5f

data/.reek.yml

@@ -77,3 +77,4 @@ detectors:
   TooManyMethods:
     exclude:
     - SidekiqUniqueJobs::Locksmith
+    - SidekiqUniqueJobs::Lock::BaseLock

data/CHANGELOG.md

@@ -13,6 +13,7 @@
 - Totally delete the hash that was growing out of proportion
 - Adds a sidekiq web extension for viewing and deleting unique digests
 - Renamed the configuration `unique:` to `lock:` (still backwards compatible)
+- Added some very simplistic conflict strategies.
 
 ## v5.1.0
 

data/README.md

@@ -9,23 +9,30 @@
 * [Support Me](#support-me)
 * [General Information](#general-information)
 * [Options](#options)
-   * [Lock Expiration](#lock-expiration)
-   * [Lock Timeout](#lock-timeout)
-   * [Unique Across Queues](#unique-across-queues)
-   * [Unique Across Workers](#unique-across-workers)
+  * [Lock Expiration](#lock-expiration)
+  * [Lock Timeout](#lock-timeout)
+  * [Unique Across Queues](#unique-across-queues)
+  * [Unique Across Workers](#unique-across-workers)
 * [Locks](#locks)
-   * [Until Executing](#until-executing)
-   * [Until Executed](#until-executed)
-   * [Until Timeout](#until-timeout)
-   * [Unique Until And While Executing](#unique-until-and-while-executing)
-   * [While Executing](#while-executing)
+  * [Until Executing](#until-executing)
+  * [Until Executed](#until-executed)
+  * [Until Timeout](#until-timeout)
+  * [Unique Until And While Executing](#unique-until-and-while-executing)
+  * [While Executing](#while-executing)
 * [Conflict Strategy](#conflict-strategy)
+  * [Log](#log)
+  * [Raise](#raise)
+  * [Reject](#reject)
+  * [Replace](#replace)
+  * [Reschedule](#reschedule)
 * [Usage](#usage)
-   * [Finer Control over Uniqueness](#finer-control-over-uniqueness)
-   * [After Unlock Callback](#after-unlock-callback)
-   * [Logging](#logging)
+  * [Finer Control over Uniqueness](#finer-control-over-uniqueness)
+  * [After Unlock Callback](#after-unlock-callback)
+  * [Logging](#logging)
 * [Debugging](#debugging)
-   * [Sidekiq Web](#sidekiq-web)
+  * [Sidekiq Web](#sidekiq-web)
+    * [Show Unique Digests](#show-unique-digests)
+    * [Show keys for digest](#show-keys-for-digest)
 * [Communication](#communication)
 * [Testing](#testing)
 * [Contributing](#contributing)
@@ -108,7 +115,7 @@ This configuration option is slightly misleading. It doesn't disregard the queue
 class Worker
   include Sidekiq::Worker
   
-  sidekiq_options: unique_across_queues: true, queue: 'default'
+  sidekiq_options unique_across_queues: true, queue: 'default'
 
   def perform(args); end
 end
@@ -124,7 +131,7 @@ This configuration option is slightly misleading. It doesn't disregard the worke
 class WorkerOne
   include Sidekiq::Worker
   
-  sidekiq_options: unique_across_workers: true, queue: 'default'
+  sidekiq_options unique_across_workers: true, queue: 'default'
 
   def perform(args); end
 end
@@ -132,7 +139,7 @@ end
 class WorkerTwo
   include Sidekiq::Worker
   
-  sidekiq_options: unique_across_workers: true, queue: 'default'
+  sidekiq_options unique_across_workers: true, queue: 'default'
 
   def perform(args); end
 end
@@ -216,6 +223,12 @@ Decides how we handle conflict. We can either reject the job to the dead queue o
 
 The last one is log which can be be used with the lock `UntilExecuted` and `UntilExpired`. Now we write a log entry saying the job could not be pushed because it is a duplicate of another job with the same arguments
 
+### Log
+
+This strategy is intended to be used with `UntilExecuted` and `UntilExpired`. It will log a line about that this is job is a duplicate of another.
+
+`sidekiq_options lock: :until_executed, on_conflict: :log`
+
 ### Raise
 
 This strategy is intended to be used with `WhileExecuting`. Basically it will allow us to let the server process crash with a specific error message and be retried without messing up the Sidekiq stats.
@@ -228,17 +241,22 @@ This strategy is intended to be used with `WhileExecuting` and will push the job
 
 `sidekiq_options lock: :while_executing, on_conflict: :reject`
 
-### Reschedule
+### Replace
 
-This strategy is intended to be used with `WhileExecuting` and will delay the job to be tried again in 5 seconds. This will mess up the sidekiq stats but will prevent exceptions from being logged and confuse your sysadmins.
+This strategy is intended to be used with client locks like `UntilExecuted`.
+It will delete any existing job for these arguments from retry, schedule and
+queue and retry the lock again. 
 
-`sidekiq_options lock: :while_executing, on_conflict: :reschedule`
+This is slightly dangerous and should probably only be used for jobs that are 
+always scheduled in the future. Currently only attempting to retry one time.
 
-### Log
+`sidekiq_options lock: :until_executed, on_conflict: :replace`
 
-This strategy is intended to be used with `UntilExecuted` and `UntilExpired`. It will log a line about that this is job is a duplicate of another.
+### Reschedule
 
-`sidekiq_options lock: :until_executed, on_conflict: :log`
+This strategy is intended to be used with `WhileExecuting` and will delay the job to be tried again in 5 seconds. This will mess up the sidekiq stats but will prevent exceptions from being logged and confuse your sysadmins.
+
+`sidekiq_options lock: :while_executing, on_conflict: :reschedule`
 
 ## Usage
 
@@ -379,6 +397,44 @@ SidekiqUniqueJobs.configure do |config|
 end
 ```
 
+If you truly wanted to test the sidekiq client push you could do something like below. Note that it will only work for the jobs that lock when the client pushes the job to redis (UntilExecuted, UntilAndWhileExecuting and UntilExpired).
+
+```ruby
+RSpec.describe Workers::CoolOne do
+  before do
+    SidekiqUniqueJobs.config.enabled = false
+  end
+
+  # ... your tests that don't test uniqueness
+
+  context 'when Sidekiq::Testing.disabled?' do
+    before do
+      Sidekiq::Testing.disable!
+      Sidekiq.redis(&:flushdb)
+    end
+
+    after do
+      Sidekiq.redis(&:flushdb)
+    end
+
+    it 'prevents duplicate jobs from being scheduled' do
+      SidekiqUniqueJobs.use_config(enabled: true) do
+        expect(described_class.perform_async(1)).not_to eq(nil)
+        expect(described_class.perform_async(1)).to eq(nil)
+      end
+    end
+  end
+end
+```
+
+I would strongly suggest you let this gem test uniqueness. If you care about how the gem is integration tested have a look at the following specs:
+
+- [spec/integration/sidekiq_unique_jobs/lock/until_and_while_executing_spec.rb](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/spec/integration/sidekiq_unique_jobs/lock/until_and_while_executing_spec.rb)
+- [spec/integration/sidekiq_unique_jobs/lock/until_executed_spec.rb](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/spec/integration/sidekiq_unique_jobs/lock/until_executed_spec.rb)
+- [spec/integration/sidekiq_unique_jobs/lock/until_expired_spec.rb](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/spec/integration/sidekiq_unique_jobs/lock/until_expired_spec.rb)
+- [spec/integration/sidekiq_unique_jobs/lock/while_executing_reject_spec.rb](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/spec/integration/sidekiq_unique_jobs/lock/while_executing_reject_spec.rb)
+- [spec/integration/sidekiq_unique_jobs/lock/while_executing_spec.rb](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/spec/integration/sidekiq_unique_jobs/lock/while_executing_spec.rb)
+
 ## Contributing
 
 1. Fork it

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -22,10 +22,12 @@ module SidekiqUniqueJobs
       #   Will call a conflict strategy if lock can't be achieved.
       # @return [String] the sidekiq job id
       def lock
+        @attempt = 0
+
         if (token = locksmith.lock(item[LOCK_TIMEOUT_KEY]))
           token
         else
-          strategy.call
+          call_strategy
         end
       end
 
@@ -62,6 +64,15 @@ module SidekiqUniqueJobs
 
       private
 
+      def call_strategy
+        @attempt += 1
+        strategy.call { lock if replace? }
+      end
+
+      def replace?
+        strategy.replace? && attempt < 2
+      end
+
       # The sidekiq job hash
       # @return [Hash] the Sidekiq job hash
       attr_reader :item
@@ -74,6 +85,10 @@ module SidekiqUniqueJobs
       # @return [Proc] the callback to use after unlock
       attr_reader :callback
 
+      # The current attempt to lock the job
+      # @return [Integer] the numerical value of the attempt
+      attr_reader :attempt
+
       # The interface to the locking mechanism
       # @return [SidekiqUniqueJobs::Locksmith]
       def locksmith

data/lib/sidekiq_unique_jobs/on_conflict.rb

@@ -5,6 +5,7 @@ require_relative 'on_conflict/null_strategy'
 require_relative 'on_conflict/log'
 require_relative 'on_conflict/raise'
 require_relative 'on_conflict/reject'
+require_relative 'on_conflict/replace'
 require_relative 'on_conflict/reschedule'
 
 module SidekiqUniqueJobs

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to raise an error on conflict
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    class Replace < OnConflict::Strategy
+      attr_reader :queue, :unique_digest
+
+      # @param [Hash] item sidekiq job hash
+      def initialize(item)
+        super
+        @queue         = item[QUEUE_KEY]
+        @unique_digest = item[UNIQUE_DIGEST_KEY]
+      end
+
+      # Replace the old job in the queue
+      # @yield to retry the lock after deleting the old one
+      def call(&block)
+        return unless delete_job_by_digest
+        delete_lock
+        block&.call
+      end
+
+      # Delete the job from either schedule, retry or the queue
+      def delete_job_by_digest
+        Scripts.call(:delete_job_by_digest, nil, keys: [queue, unique_digest])
+      end
+
+      # Delete the keys belonging to the job
+      def delete_lock
+        Scripts.call(:delete_by_digest, nil, keys: [UNIQUE_SET, unique_digest])
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb

@@ -23,6 +23,10 @@ module SidekiqUniqueJobs
       def call
         fail NotImplementedError, 'needs to be implemented in child class'
       end
+
+      def replace?
+        is_a?(Replace)
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  VERSION = '6.0.0.rc7'
+  VERSION = '6.0.0.rc8'
 end

data/redis/delete_by_digest.lua

@@ -11,7 +11,7 @@ local run_grabbed_key   = unique_digest .. ':RUN:GRABBED'
 local run_available_key = unique_digest .. ':RUN:AVAILABLE'
 local run_version_key   = unique_digest .. ':RUN:VERSION'
 
-redis.call('SREM', unique_keys, unique_digest)
+local count = redis.call('SREM', unique_keys, unique_digest)
 redis.call('DEL', exists_key)
 redis.call('DEL', grabbed_key)
 redis.call('DEL', available_key)
@@ -20,3 +20,5 @@ redis.call('DEL', run_exists_key)
 redis.call('DEL', run_grabbed_key)
 redis.call('DEL', run_available_key)
 redis.call('DEL', run_version_key)
+
+return count

data/redis/delete_job_by_digest.lua

@@ -0,0 +1,58 @@
+local queue         = "queue:" .. KEYS[1]
+local unique_digest = KEYS[2]
+
+local function delete_from_sorted_set(name, digest)
+  local per   = 50
+  local total = redis.call('zcard', name)
+  local index = 0
+  local result
+  -- redis.log(redis.LOG_DEBUG, "delete_from_sorted_set("..name..","..digest..")")
+  while (index < total) do
+    -- redis.log(redis.LOG_DEBUG, "delete_from_sorted_set("..name..","..digest..") - "..index.."-"..per)
+    local items = redis.call('ZRANGE', name, index, index + per -1)
+    for _, item in pairs(items) do
+      -- redis.log(redis.LOG_DEBUG, "delete_from_sorted_set("..name..","..digest..") - current item: " .. item)
+      if string.find(item, digest) then
+        -- redis.log(redis.LOG_DEBUG, "delete_from_sorted_set("..name..","..digest..") - deleting item")
+        redis.call('ZREM', name, item)
+        result = item
+        break
+      end
+    end
+    index = index + per
+  end
+  return result
+end
+
+local per     = 50
+local total   = redis.call('LLEN', queue)
+local index   = 0
+local result  = nil
+
+-- redis.log(redis.LOG_DEBUG, "delete_job_by_digest.lua - looping through: " .. queue)
+while (index < total) do
+  -- redis.log(redis.LOG_DEBUG, "delete_job_by_digest.lua - " .. index .. "-" .. per)
+  local items = redis.call('LRANGE', queue, index, index + per -1)
+  for _, item in pairs(items) do
+    -- redis.log(redis.LOG_DEBUG, "delete_job_by_digest.lua - item: " .. item)
+    if string.find(item, unique_digest) then
+      -- redis.log(redis.LOG_DEBUG, "delete_job_by_digest.lua - found item with digest: " .. unique_digest .. " in: " ..queue)
+      redis.call('LREM', queue, 1, item)
+      result = item
+      break
+    end
+  end
+  index = index + per
+end
+
+if result then
+  return result
+end
+
+result = delete_from_sorted_set('schedule', unique_digest)
+if result then
+  return result
+end
+
+result = delete_from_sorted_set('retry', unique_digest)
+return result

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 6.0.0.rc7
+  version: 6.0.0.rc8
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-23 00:00:00.000000000 Z
+date: 2018-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -289,6 +289,7 @@ files:
 - lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb
 - lib/sidekiq_unique_jobs/on_conflict/raise.rb
 - lib/sidekiq_unique_jobs/on_conflict/reject.rb
+- lib/sidekiq_unique_jobs/on_conflict/replace.rb
 - lib/sidekiq_unique_jobs/on_conflict/reschedule.rb
 - lib/sidekiq_unique_jobs/on_conflict/strategy.rb
 - lib/sidekiq_unique_jobs/options_with_fallback.rb
@@ -311,6 +312,7 @@ files:
 - redis/create.lua
 - redis/delete.lua
 - redis/delete_by_digest.lua
+- redis/delete_job_by_digest.lua
 - redis/release_lock.lua
 - redis/release_stale_locks.lua
 - redis/signal.lua