sidekiq-unique-jobs 3.0.14 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 47d33808dbdda5aed24c7559888c87e9314feda9
-  data.tar.gz: 7e0698a1ddb3ac9ecff610ec1837e7cbeef6f844
+  metadata.gz: 9335a1bcf7592b094855e9d4562840691c9bcc96
+  data.tar.gz: bdd063a97f2a10c1844dda8c51b42221b50a52b6
 SHA512:
-  metadata.gz: 31028d7ac6a86361d3e32e4946223d0803fa71111e8b4e080947e3a9774a5935e57f5fe9661e63fdbc864ff214c8590548a5e12fce4058f3fad8b68358b2efd0
-  data.tar.gz: f0a3f8aed7f244ca06067c2753f508e3349dbcd65cce0f7acfdad57d29fbcfafa310b5ff50315a6b3258de39c7e37ffa72dc742924cb7e229f1775492ce6266c
+  metadata.gz: 2fd7ce95d41c91ec3155336cf7961e1d6986e92a261e8bde242ed2a720338abff103754c7973cad85d3a29b2f69d13234379877f64b466498ff904d86c6b0729
+  data.tar.gz: 7b3488a1eba8ff8b2457d5f70b1253dbdffff3bfca249cee54bf4f6a2673360ea3ce22b573c1297d5ee59959a430f9408e385f9cc4c6bb448966f0b16ef2b3fb

data/.editorconfig

@@ -0,0 +1,14 @@
+# This file is for unifying the coding style for different editors and IDEs
+# editorconfig.org
+
+root = true
+
+[*]
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = true

data/.gitignore

@@ -9,3 +9,5 @@ Gemfile.lock
 .idea/
 gemfiles/*.lock
 *.sw?
+coverage/
+tmp/

data/.rubocop.yml

@@ -1,3 +1,11 @@
+AllCops:
+  Include:
+    - '**/Rakefile'
+    - '**/lib'
+    - '**/spec'
+  Exclude:
+    - 'bin/**/*'
+    - 'gemfiles/**/*'
 
 Lint/HandleExceptions:
   Enabled: true

data/.simplecov

@@ -0,0 +1,12 @@
+require 'simplecov-json'
+
+SimpleCov.refuse_coverage_drop
+SimpleCov.formatters = [
+  SimpleCov::Formatter::HTMLFormatter,
+  SimpleCov::Formatter::JSONFormatter
+]
+SimpleCov.start do
+  add_filter '/spec/'
+  add_filter '/bin/'
+  add_filter '/gemfiles/'
+end

data/.travis.yml

@@ -1,12 +1,19 @@
+sudo: false
 language: ruby
+cache: bundler
 services:
   - redis-server
+script:
+  - if [[ "${STYLE}" = "true" ]]; then bundle exec rubocop; fi;
+  - bundle exec rspec spec
 rvm:
   - jruby-19mode
-  - rbx2
+  - jruby-9.0.0.0
+  - rbx-2
   - 2.0.0
-  - 2.1.2
-  - 2.2.2
+  - 2.1.7
+  - 2.2.3
+env: STYLE=false
 gemfile:
   - gemfiles/sidekiq_develop.gemfile
   - gemfiles/sidekiq_2.17.gemfile
@@ -14,13 +21,14 @@ gemfile:
   - gemfiles/sidekiq_3.1.gemfile
   - gemfiles/sidekiq_3.2.gemfile
   - gemfiles/sidekiq_3.3.gemfile
-branches:
-  only:
-    - master
+
 notifications:
   email:
     recipients:
       - mikael@zoolutions.se
 matrix:
-  allow_failures:
-    - rvm: rbx2
+  fast_finish: true
+  include:
+    - rvm: 2.2.3
+      gemfile: gemfiles/sidekiq_3.3.gemfile
+      env: STYLE=true

data/Appraisals

@@ -1,23 +1,23 @@
-appraise "sidekiq-develop" do
+appraise 'sidekiq-develop' do
   gem 'sidekiq', github: 'mperham/sidekiq'
 end
 
-appraise "sidekiq-2.17" do
+appraise 'sidekiq-2.17' do
   gem 'sidekiq', '~> 2.17.0'
 end
 
-appraise "sidekiq-3.0" do
-  gem "sidekiq", '~> 3.0.0'
+appraise 'sidekiq-3.0' do
+  gem 'sidekiq', '~> 3.0.0'
 end
 
-appraise "sidekiq-3.1" do
-  gem "sidekiq", '~> 3.1.0'
+appraise 'sidekiq-3.1' do
+  gem 'sidekiq', '~> 3.1.0'
 end
 
-appraise "sidekiq-3.2" do
-  gem "sidekiq", '~> 3.2.0'
+appraise 'sidekiq-3.2' do
+  gem 'sidekiq', '~> 3.2.0'
 end
 
-appraise "sidekiq-3.3" do
-  gem "sidekiq", '~> 3.3.0'
+appraise 'sidekiq-3.3' do
+  gem 'sidekiq', '~> 3.3.0'
 end

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## v3.0.16 (Unreleased)
+
+- Improved uniqueness handling (complete refactoring, upgrade with causion)
+- 100% breaking changes
+
+## v3.0.15
+- Jobs only ever unlock themselves now (see #96 & #94 for info) thanks @pik
+- Slight refactoring and internal renaming. Shouldn't affect anyone
+- Run locks as an alternative when you only need to prevent the same job running twice but want to be able to schedule it multiple times. See #99 (thanks @pik)
+- Fixes #90, #92, #93, #97, #98, #100, #101, #105
+
 ## v3.0.14
 - Improve uniqueness check performance thanks @mpherham
 - Remove locks in sidekiq fake testing mode

data/Gemfile

@@ -2,4 +2,14 @@ source 'http://rubygems.org'
 gemspec
 
 gem 'appraisal', '~> 2.0.0'
-gem 'pry', platform: :mri
+
+gem 'rspec-its',     require: false
+
+platform :mri do
+  gem 'pry',                       require: false
+  gem 'pry-rescue',                require: false
+  gem 'pry-byebug',                require: false
+  gem 'simplecov-json',            require: false
+  gem 'memory_profiler',           require: false
+  gem 'codeclimate-test-reporter', require: false
+end

data/README.md

@@ -6,6 +6,13 @@ The missing unique jobs for sidekiq
 
 See https://github.com/mperham/sidekiq#requirements for what is required. Starting from 3.0.13 only sidekiq 3 is supported and support for MRI 1.9 is dropped (it might work but won't be worked on)
 
+Version 4 requires redis 2.6.2!! Don't upgrade to version 4 unless you are on redis 2.6.2.
+
+## Upgrade instructions
+
+Easy path - Drop all your unique jobs before upgrading the gem!
+Hard path - See above... Start with a clean slate :)
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -20,6 +27,34 @@ Or install it yourself as:
 
     $ gem install sidekiq-unique-jobs
 
+## A word on locking
+
+Like @mperham mentions on (this wiki page)[https://github.com/mperham/sidekiq/wiki/Related-Projects#unique-jobs] it is hard to enforce uniqueness with redis in a distributed redis setting. 
+
+To make things worse there are many ways of wanting to enforce uniqueness. 
+
+### While Executing
+
+Is to make sure that a job can be scheduled any number of times but only executed a single time per argument provided to the job we call this runtime uniqueness. This is probably most useful forbackground jobs that are fast to execute. (See mhenrixon/sidekiq-unique-jobs#111 for a great example of when this would be right.) While the job is executing/performing no other jobs can be executed at the same time.
+
+### Until Executing
+
+This means that a job can only be scheduled into redis once per whatever the configuration of unique arguments. Any jobs added until the first one of the same arguments has been unlocked will just be dropped. This is what was tripping many people up. They would schedule a job to run in the future and it would be impossible to schedule new jobs with those same arguments even immediately. There was some forth and back between also locking jobs on the scheduled queue and the regular queues but in the end I decided it was best to separate these two features out into different locking mechanisms. I think what most people are after is to be able to lock a job while executing or that seems to be what people are most missing at the moment.
+
+### Until Executed
+
+This is the combination of the two above. First we lock the job until it executes, then as the job begins executes we keep the lock so that no other jobs with the same arguments can execute at the same time.
+
+### Until Timeout
+
+The job won't be unlocked until the timeout/expiry runs out.
+
+### Uniqueness Scope
+
+- Queue specific locks
+- Across all queues.
+- Timed / Scheduled jobs
+
 ## Usage
 
 All that is required is that you specifically set the sidekiq option for *unique* to true like below:
@@ -33,12 +68,12 @@ should be unique. The job will be unique for the number of seconds configured (d
 or until the job has been completed. Thus, the job will be unique for the shorter of the two.  Note that Sidekiq versions before 3.0 will remove job keys after an hour, which means jobs can remain unique for at most an hour.
 
 *If you want the unique job to stick around even after it has been successfully
-processed then just set the unique_unlock_order to anything except `:before_yield` or `:after_yield` (`unique_unlock_order = :never`)
+processed then just set the unique_lock to anything except `:before_yield` or `:after_yield` (`unique_lock = :until_timeout`)
 
 You can also control the expiration length of the uniqueness check. If you want to enforce uniqueness over a longer period than the default of 30 minutes then you can pass the number of seconds you want to use to the sidekiq options:
 
 ```ruby
-sidekiq_options unique: true, unique_job_expiration: 120 * 60 # 2 hours
+sidekiq_options unique: true, unique_expiration: 120 * 60 # 2 hours
 ```
 
 Requiring the gem in your gemfile should be sufficient to enable unique jobs.
@@ -96,19 +131,36 @@ Note that objects passed into workers are converted to JSON *after* running thro
 
 ### Unlock Ordering
 
-By default the server middleware will release the worker lock after yielding to the next middleware or worker. Alternatively, this can be changed by passing the `unique_unlock_order` option:
+By default the server middleware will release the worker lock after yielding to the next middleware or worker. Alternatively, this can be changed by passing the `unique_lock` option:
 
 ```ruby
 class UniqueJobWithFilterMethod
   include Sidekiq::Worker
   sidekiq_options unique: true,
-                  unique_unlock_order: :before_yield
+                  unique_locks: :until_executing
 
   ...
 
 end
 ```
 
+### After Unlock Callback
+
+If you are using :after_yield as your unlock ordering, Unique Job offers a callback to perform some work after the block is yielded.
+
+```ruby
+class UniqueJobWithFilterMethod
+  include Sidekiq::Worker
+  sidekiq_options unique: true,
+
+  def after_unlock
+   # block has yielded and lock is released
+  end
+  ...
+end.
+
+```
+
 ### Unique Storage Method
 
 Starting from sidekiq-unique-jobs 3.0.14 we will use the `set` method in a way that has been available since redis 2.6.12. If you are on an older redis version you will have to change a config value like below.
@@ -137,6 +189,8 @@ end
 
 ### Testing
 
+To enable the testing for `sidekiq-unique-jobs`, add `require 'sidekiq_unique_jobs/testing'` to your testing helper.
+
 SidekiqUniqueJobs uses mock_redis for inline testing. Due to complaints about having that as a runtime dependency it was made a development dependency so if you are relying on inline testing you will have to add `gem 'mock_redis'` to your Gemfile.
 
 ## Contributing

data/Rakefile

@@ -1,4 +1,5 @@
 #!/usr/bin/env rake
+
 require 'rubygems'
 require 'bundler/setup'
 require 'bundler/gem_tasks'
@@ -8,4 +9,4 @@ require 'rubocop/rake_task'
 RuboCop::RakeTask.new(:style)
 RSpec::Core::RakeTask.new(:spec)
 
-task default: [:spec, :style]
+task default: [:style, :spec]

data/circle.yml

@@ -0,0 +1,36 @@
+machine:
+  ruby: 2.2.3
+  java:
+    version: oraclejdk8
+  environment:
+    JRUBY_OPTS: '-J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-noverify -X-C -Xcompile.invokedynamic=false --1.9 -J-Xmx1g'
+
+dependencies:
+  pre:
+    - rvm install jruby-9.0.0.0
+    - rvm install jruby-1.7
+    - rvm install rbx-2
+    - rvm install 2.0.0
+    - rvm install 2.1.7
+  override:
+    - rvm-exec jruby-9.0.0.0 bundle install
+    - rvm-exec jruby-1.7 bundle install
+    - rvm-exec rbx-2 bundle install
+    - rvm-exec 2.0.0 bundle install
+    - rvm-exec 2.1.7 bundle install
+    - rvm-exec 2.2.3 bundle install
+    - rvm-exec jruby-9.0.0.0 bundle exec appraisal install
+    - rvm-exec jruby-1.7 bundle exec appraisal install
+    - rvm-exec rbx-2 bundle exec appraisal install
+    - rvm-exec 2.0.0 bundle exec appraisal install
+    - rvm-exec 2.1.7 bundle exec appraisal install
+    - rvm-exec 2.2.3 bundle exec appraisal install
+
+test:
+  override:
+    - rvm-exec jruby-9.0.0.0 bundle exec appraisal rspec
+    - rvm-exec jruby-1.7 bundle exec appraisal rspec
+    - rvm-exec rbx-2 bundle exec appraisal rspec
+    - rvm-exec 2.0.0 bundle exec appraisal rspec
+    - rvm-exec 2.1.7 bundle exec appraisal rspec
+    - rvm-exec 2.2.3 bundle exec appraisal rspec

data/gemfiles/sidekiq_2.17.gemfile

@@ -3,7 +3,14 @@
 source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
-gem "pry", :platform => :mri
 gem "sidekiq", "~> 2.17.0"
 
+platforms :mri do
+  gem "pry-suite", :require => false
+  gem "let_it_go", :require => false
+  gem "memory-profiler", :require => false
+  gem "simplecov-json", :require => false
+  gem "codeclimate-test-reporter", :require => false
+end
+
 gemspec :path => "../"

data/gemfiles/sidekiq_3.0.gemfile

@@ -3,7 +3,14 @@
 source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
-gem "pry", :platform => :mri
 gem "sidekiq", "~> 3.0.0"
 
+platforms :mri do
+  gem "pry-suite", :require => false
+  gem "let_it_go", :require => false
+  gem "memory-profiler", :require => false
+  gem "simplecov-json", :require => false
+  gem "codeclimate-test-reporter", :require => false
+end
+
 gemspec :path => "../"

data/gemfiles/sidekiq_3.1.gemfile

@@ -3,7 +3,14 @@
 source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
-gem "pry", :platform => :mri
 gem "sidekiq", "~> 3.1.0"
 
+platforms :mri do
+  gem "pry-suite", :require => false
+  gem "let_it_go", :require => false
+  gem "memory-profiler", :require => false
+  gem "simplecov-json", :require => false
+  gem "codeclimate-test-reporter", :require => false
+end
+
 gemspec :path => "../"

data/gemfiles/sidekiq_3.2.gemfile

@@ -3,7 +3,14 @@
 source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
-gem "pry", :platform => :mri
 gem "sidekiq", "~> 3.2.0"
 
+platforms :mri do
+  gem "pry-suite", :require => false
+  gem "let_it_go", :require => false
+  gem "memory-profiler", :require => false
+  gem "simplecov-json", :require => false
+  gem "codeclimate-test-reporter", :require => false
+end
+
 gemspec :path => "../"

data/gemfiles/sidekiq_3.3.gemfile

@@ -3,7 +3,14 @@
 source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
-gem "pry", :platform => :mri
 gem "sidekiq", "~> 3.3.0"
 
+platforms :mri do
+  gem "pry-suite", :require => false
+  gem "let_it_go", :require => false
+  gem "memory-profiler", :require => false
+  gem "simplecov-json", :require => false
+  gem "codeclimate-test-reporter", :require => false
+end
+
 gemspec :path => "../"

data/gemfiles/sidekiq_develop.gemfile

@@ -3,7 +3,14 @@
 source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
-gem "pry", :platform => :mri
 gem "sidekiq", :github => "mperham/sidekiq"
 
+platforms :mri do
+  gem "pry-suite", :require => false
+  gem "let_it_go", :require => false
+  gem "memory-profiler", :require => false
+  gem "simplecov-json", :require => false
+  gem "codeclimate-test-reporter", :require => false
+end
+
 gemspec :path => "../"

data/lib/sidekiq-unique-jobs.rb

@@ -1,9 +1,13 @@
-# rubocop:disable FileName
-require 'yaml' if RUBY_VERSION.include?('2.0.0')
+require 'yaml' if RUBY_VERSION.include?('2.0.0') # rubocop:disable FileName
+require 'sidekiq_unique_jobs/core_ext'
+require 'sidekiq_unique_jobs/options_with_fallback'
+require 'sidekiq_unique_jobs/scripts'
+require 'sidekiq_unique_jobs/unique_args'
+require 'sidekiq_unique_jobs/unlockable'
+require 'sidekiq_unique_jobs/lock'
 require 'sidekiq_unique_jobs/middleware'
 require 'sidekiq_unique_jobs/version'
 require 'sidekiq_unique_jobs/config'
-require 'sidekiq_unique_jobs/payload_helper'
 require 'sidekiq_unique_jobs/sidekiq_unique_ext'
 
 require 'ostruct'
@@ -13,11 +17,11 @@ module SidekiqUniqueJobs
 
   def config
     @config ||= Config.new(
-      unique_prefix: 'sidekiq_unique',
-      unique_args_enabled: false,
+      unique_prefix: 'uniquejobs',
+      unique_args_enabled: true,
       default_expiration: 30 * 60,
-      default_unlock_order: :after_yield,
-      unique_storage_method: :new
+      default_lock: :while_executing,
+      redis_test_mode: :redis # :mock
     )
   end
 
@@ -25,8 +29,22 @@ module SidekiqUniqueJobs
     config.unique_args_enabled
   end
 
-  def configure
-    yield config
+  def default_lock
+    config.default_lock
+  end
+
+  def configure(options = {})
+    if block_given?
+      yield config
+    else
+      options.each do |key, val|
+        config[key] = val
+      end
+    end
+  end
+
+  def namespace
+    @namespace ||= Sidekiq.redis { |c| c.respond_to?(:namespace) ? c.namespace : nil }
   end
 
   # Attempt to constantize a string worker_class argument, always
@@ -37,4 +55,21 @@ module SidekiqUniqueJobs
   rescue NameError
     worker_class
   end
+
+  def redis_version
+    @redis_version ||= Sidekiq.redis { |c| c.info('server')['redis_version'] }
+  end
+
+  def connection(redis_pool = nil, &block)
+    return mock_redis if config.mocking?
+    redis_pool ? redis_pool.with(&block) : Sidekiq.redis(&block)
+  end
+
+  def mock_redis
+    @redis_mock ||= MockRedis.new if defined?(MockRedis)
+  end
+
+  def synchronize(item, redis_pool, &blk)
+    Lock::WhileExecuting.synchronize(item, redis_pool, &blk)
+  end
 end