sidekiq-unique-jobs 3.0.13 → 3.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b074cf18e3e7d0878e69527a8716e0c962cf9279
-  data.tar.gz: b960227a096e842d45446eb2e44d916b638451a5
+  metadata.gz: 47d33808dbdda5aed24c7559888c87e9314feda9
+  data.tar.gz: 7e0698a1ddb3ac9ecff610ec1837e7cbeef6f844
 SHA512:
-  metadata.gz: aaaf71ae3c669a2cb77390e1d4c4a9fd12ef0cb1909f187cc9435ff858b6c2967a2e5f64f15c2b129f2776b9c1dd112bc1d44bdb0da114cdcd38aa0a8adf7b50
-  data.tar.gz: 985a07f514bbc295c1ef2191eacde190246c0d6a7368f1db82046e77a487a7bc1207edfda5c485d4775a9d9d0592ef1e97c937384ff8979c069a732179e10e27
+  metadata.gz: 31028d7ac6a86361d3e32e4946223d0803fa71111e8b4e080947e3a9774a5935e57f5fe9661e63fdbc864ff214c8590548a5e12fce4058f3fad8b68358b2efd0
+  data.tar.gz: f0a3f8aed7f244ca06067c2753f508e3349dbcd65cce0f7acfdad57d29fbcfafa310b5ff50315a6b3258de39c7e37ffa72dc742924cb7e229f1775492ce6266c

data/.travis.yml

@@ -8,11 +8,12 @@ rvm:
   - 2.1.2
   - 2.2.2
 gemfile:
+  - gemfiles/sidekiq_develop.gemfile
+  - gemfiles/sidekiq_2.17.gemfile
   - gemfiles/sidekiq_3.0.gemfile
   - gemfiles/sidekiq_3.1.gemfile
   - gemfiles/sidekiq_3.2.gemfile
   - gemfiles/sidekiq_3.3.gemfile
-  - gemfiles/sidekiq_develop.gemfile
 branches:
   only:
     - master

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## v3.0.14
+- Improve uniqueness check performance thanks @mpherham
+- Remove locks in sidekiq fake testing mode
+- Do not unlock jobs when sidekiq is shutting down 
+
 ## v3.0.13
 - Improved testing capabilities (testing uniqueness should not work better)
 - Configurable logging of duplicate payloads

data/README.md

@@ -43,6 +43,19 @@ sidekiq_options unique: true, unique_job_expiration: 120 * 60 # 2 hours
 
 Requiring the gem in your gemfile should be sufficient to enable unique jobs.
 
+### Usage with ActiveJob
+
+```ruby
+Sidekiq.default_worker_options = {
+   'unique' => true,
+   'unique_args' => proc do |args|
+     [args.first.except('job_id')]
+   end
+}
+SidekiqUniqueJobs.config.unique_args_enabled = true
+```
+
+
 ### Finer Control over Uniqueness
 
 Sometimes it is desired to have a finer control over which arguments are used in determining uniqueness of the job, and others may be _transient_. For this use-case, you need to set `SidekiqUniqueJobs.config.unique_args_enabled` to true in an initializer, and then defined either `unique_args` method, or a ruby proc.
@@ -96,6 +109,17 @@ class UniqueJobWithFilterMethod
 end
 ```
 
+### Unique Storage Method
+
+Starting from sidekiq-unique-jobs 3.0.14 we will use the `set` method in a way that has been available since redis 2.6.12. If you are on an older redis version you will have to change a config value like below.
+
+```ruby
+SidekiqUniqueJobs.config.unique_storage_method = :old
+```
+
+That should allow you to keep using redis in the old fashion way. See #89 for mor information.
+
+
 ### Logging
 
 To see logging in sidekiq when duplicate payload has been filtered out you can enable on a per worker basis using the sidekiq options.  The default value is false

data/gemfiles/sidekiq_2.17.gemfile

@@ -2,7 +2,7 @@
 
 source "http://rubygems.org"
 
-gem "appraisal", "~> 1.0.0"
+gem "appraisal", "~> 2.0.0"
 gem "pry", :platform => :mri
 gem "sidekiq", "~> 2.17.0"
 

data/lib/sidekiq-unique-jobs.rb

@@ -16,7 +16,8 @@ module SidekiqUniqueJobs
       unique_prefix: 'sidekiq_unique',
       unique_args_enabled: false,
       default_expiration: 30 * 60,
-      default_unlock_order: :after_yield
+      default_unlock_order: :after_yield,
+      unique_storage_method: :new
     )
   end
 

data/lib/sidekiq_unique_jobs/config.rb

@@ -4,7 +4,8 @@ module SidekiqUniqueJobs
       :unique_prefix,
       :unique_args_enabled,
       :default_expiration,
-      :default_unlock_order
+      :default_unlock_order,
+      :unique_storage_method
     ]
 
     class << self

data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb

@@ -17,7 +17,9 @@ module SidekiqUniqueJobs
           end
 
           def self.review(worker_class, item, queue, redis_pool = nil, log_duplicate_payload = false)
-            new(worker_class, item, queue, redis_pool, log_duplicate_payload).review { yield }
+            new(worker_class, item, queue, redis_pool, log_duplicate_payload).review do
+              yield
+            end
           end
 
           def initialize(worker_class, item, queue, redis_pool = nil, log_duplicate_payload = false)
@@ -31,12 +33,12 @@ module SidekiqUniqueJobs
 
           def review
             item['unique_hash'] = payload_hash
+
             unless unique_for_connection?
-              if @log_duplicate_payload
-                Sidekiq.logger.warn "payload is not unique #{item}"
-              end
+              Sidekiq.logger.warn "payload is not unique #{item}" if @log_duplicate_payload
               return
             end
+
             yield
           end
 
@@ -44,23 +46,24 @@ module SidekiqUniqueJobs
 
           attr_reader :item, :worker_class, :redis_pool, :queue, :log_duplicate_payload
 
-          # rubocop:disable MethodLength
           def unique_for_connection?
-            unique = false
+            send("#{storage_method}_unique_for?")
+          end
+
+          def storage_method
+            SidekiqUniqueJobs.config.unique_storage_method
+          end
+
+          def old_unique_for?
+            unique = nil
             connection do |conn|
               conn.watch(payload_hash)
-
-              if conn.get(payload_hash).to_i == 1 ||
-                 (conn.get(payload_hash).to_i == 2 && item['at'])
+              pid = conn.get(payload_hash).to_i
+              if pid == 1 || (pid == 2 && item['at'])
                 # if the job is already queued, or is already scheduled and
                 # we're trying to schedule again, abort
                 conn.unwatch
               else
-                # if the job was previously scheduled and is now being queued,
-                # or we've never seen it before
-                expires_at = unique_job_expiration || SidekiqUniqueJobs.config.default_expiration
-                expires_at = ((Time.at(item['at']) - Time.now.utc) + expires_at).to_i if item['at']
-
                 unique = conn.multi do
                   # set value of 2 for scheduled jobs, 1 for queued jobs.
                   conn.setex(payload_hash, expires_at, item['at'] ? 2 : 1)
@@ -69,7 +72,20 @@ module SidekiqUniqueJobs
             end
             unique
           end
-          # rubocop:enable MethodLength
+
+          def new_unique_for?
+            connection do |conn|
+              return conn.set(payload_hash, item['at'] ? 2 : 1, nx: true, ex: expires_at)
+            end
+          end
+
+          def expires_at
+            # if the job was previously scheduled and is now being queued,
+            # or we've never seen it before
+            ex = unique_job_expiration || SidekiqUniqueJobs.config.default_expiration
+            ex = ((Time.at(item['at']) - Time.now.utc) + ex).to_i if item['at']
+            ex
+          end
 
           def connection(&block)
             SidekiqUniqueJobs::Connectors.connection(redis_pool, &block)

data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb

@@ -8,17 +8,17 @@ module SidekiqUniqueJobs
         attr_reader :unlock_order, :redis_pool
 
         def call(worker, item, _queue, redis_pool = nil)
+          operative = true
           @redis_pool = redis_pool
-
           decide_unlock_order(worker.class)
           lock_key = payload_hash(item)
-          unlocked = before_yield? ? unlock(lock_key).inspect : 0
-
+          unlock(lock_key) if before_yield?
           yield
+        rescue Sidekiq::Shutdown
+          operative = false
+          raise
         ensure
-          if after_yield? || !defined? unlocked || unlocked != 1
-            unlock(lock_key)
-          end
+          unlock(lock_key) if after_yield? && operative
         end
 
         def decide_unlock_order(klass)

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,3 +1,3 @@
 module SidekiqUniqueJobs
-  VERSION = '3.0.13'
+  VERSION = '3.0.14'
 end

data/spec/lib/client_spec.rb

@@ -57,7 +57,7 @@ describe 'Client' do
 
     it 'enqueues previously scheduled job' do
       QueueWorker.sidekiq_options unique: true
-      QueueWorker.perform_in(60 * 60, 1, 2)
+      QueueWorker.perform_in(60 * 60, [1, 2])
 
       # time passes and the job is pulled off the schedule:
       Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue', 'args' => [1, 2])

data/spec/lib/middleware/server/unique_jobs_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'sidekiq/cli'
 
 module SidekiqUniqueJobs
   module Middleware
@@ -20,8 +21,10 @@ module SidekiqUniqueJobs
           end
 
           it 'returns true when unique_unlock_order has been set' do
-            UniqueWorker.sidekiq_options unique_unlock_order: :before_yield
-            expect(subject.unlock_order_configured?(UniqueWorker))
+            test_worker_class = UniqueWorker.dup
+            test_worker_class.sidekiq_options unique_unlock_order: :before_yield
+
+            expect(subject.unlock_order_configured?(test_worker_class))
               .to eq(true)
           end
         end
@@ -29,9 +32,11 @@ module SidekiqUniqueJobs
         describe '#decide_unlock_order' do
           context 'when worker has specified unique_unlock_order' do
             it 'changes unlock_order to the configured value' do
-              UniqueWorker.sidekiq_options unique_unlock_order: :before_yield
+              test_worker_class = UniqueWorker.dup
+              test_worker_class.sidekiq_options unique_unlock_order: :before_yield
+
               expect do
-                subject.decide_unlock_order(UniqueWorker)
+                subject.decide_unlock_order(test_worker_class)
               end.to change { subject.unlock_order }.to :before_yield
             end
           end
@@ -39,6 +44,7 @@ module SidekiqUniqueJobs
           context "when worker hasn't specified unique_unlock_order" do
             it 'falls back to configured default_unlock_order' do
               SidekiqUniqueJobs.config.default_unlock_order = :before_yield
+
               expect do
                 subject.decide_unlock_order(UniqueWorker)
               end.to change { subject.unlock_order }.to :before_yield
@@ -75,6 +81,31 @@ module SidekiqUniqueJobs
             expect(subject.default_unlock_order).to eq(:after_yield)
           end
         end
+
+        describe '#call' do
+          context 'unlock' do
+            let(:uj) { SidekiqUniqueJobs::Middleware::Server::UniqueJobs.new }
+            let(:items) { [AfterYieldWorker.new, { 'class' => 'testClass' }, 'test'] }
+
+            it 'should unlock after yield when call succeeds' do
+              expect(uj).to receive(:unlock)
+
+              uj.call(*items) { true }
+            end
+
+            it 'should unlock after yield when call errors' do
+              expect(uj).to receive(:unlock)
+
+              expect { uj.call(*items) { fail } }.to raise_error(RuntimeError)
+            end
+
+            it 'should not unlock after yield on shutdown, but still raise error' do
+              expect(uj).to_not receive(:unlock)
+
+              expect { uj.call(*items) { fail Sidekiq::Shutdown } }.to raise_error(Sidekiq::Shutdown)
+            end
+          end
+        end
       end
     end
   end

data/spec/support/after_yield_worker.rb

@@ -0,0 +1,13 @@
+class AfterYieldWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :working, retry: 1, backtrace: 10, unique_unlock_order: :after_yield
+  sidekiq_options unique: false
+
+  sidekiq_retries_exhausted do |msg|
+    Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
+  end
+
+  def perform(*)
+    # NO-OP
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 3.0.13
+  version: 3.0.14
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-05 00:00:00.000000000 Z
+date: 2015-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -170,6 +170,7 @@ files:
 - spec/lib/sidekiq_unique_ext_spec.rb
 - spec/lib/unlock_order_spec.rb
 - spec/spec_helper.rb
+- spec/support/after_yield_worker.rb
 - spec/support/another_unique_worker.rb
 - spec/support/my_worker.rb
 - spec/support/sidekiq_meta.rb