sidekiq-unique-jobs 4.0.17 → 4.0.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed1a8ffb2ff6f320e5304d4d205b8e1b0d45e98b
-  data.tar.gz: a65deff79a35909d35259361b430211bb30fb65a
+  metadata.gz: 5e48826d7669ceaa951fae20cf68de7a3e9676b9
+  data.tar.gz: 42a874852687d4bba62d79fee6adc3119aec6ebe
 SHA512:
-  metadata.gz: 5e88464e143a66dd207f509b1739bb7d838f08131694ce3c7dca2d0eae16d10e751c24731a7e41f1a1ea68085c9850326a16685e6c254726d778cbf89bdbf13f
-  data.tar.gz: 90416313c946c6bbf96cfe2c6399714d2720254755db8d4505c24bfd6e4f7e9aac364487c6dc5ffba9cd64947bfd9cbee9472ff9d197606395b05d9871200165
+  metadata.gz: a40449737349637916e6498dd0cb2408ec9f9300d3cf0f900effdfc290f4148a1acc672fcd0e85932d9d9198d5102807b103ec768346beeebe7cfdce06fabad7
+  data.tar.gz: 2ac854b6034970cb0609a9f855f576939d31e3dacfa8627bc4bbbccbed8d08e98e41ca478a29103e0991da599f7fc0378a6dd34f6ba7fcaca15a43751e580b14

data/.dockerignore

@@ -0,0 +1,4 @@
+tmp/restart.txt
+tmp/cache
+tmp/pids
+tmp/sockets

data/.gitignore

@@ -11,3 +11,4 @@ gemfiles/*.lock
 *.sw?
 coverage/
 tmp/
+rails_example/spec/examples.txt

data/.travis.yml

@@ -13,10 +13,10 @@ script:
 rvm:
   - jruby-19mode
   - jruby-9.0.4.0
-  - rbx-2
+  - jruby-9.1.2.0
+  - rbx-3.9
   - 2.1.8
   - 2.2.4
-  - 2.3.0
 env: STYLE=false
 gemfile:
   - gemfiles/sidekiq_develop.gemfile
@@ -35,7 +35,7 @@ notifications:
 matrix:
   fast_finish: true
   include:
-    - rvm: 2.3.0
+    - rvm: 2.3.1
       gemfile: gemfiles/sidekiq_develop.gemfile
       env: STYLE=true
 addons:

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## v4.0.18
+
+- Allow mock_redis to be used over redis
+- Fixes some locking inconsistencies
+
+## v4.0.17
+
+- Always release lock in the lua script `release_lock.lua` and return success [#169](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/169)
+
 ## v4.0.16
 
 - Allow run & queue lock timeout (expiration) to be different [#164](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/164)

data/Gemfile

@@ -4,6 +4,7 @@ gemspec
 gem 'appraisal', '~> 2.0.0'
 gem 'rspec-its',  require: false
 gem 'rspec-wait', require: false
+gem 'mock_redis', require: false
 
 platforms :mri_23 do
   gem 'fasterer',                  require: false

data/README.md

@@ -43,7 +43,7 @@ Due to discoverability of the different lock types `unique` sidekiq option it wa
 sidekiq_options unique: :while_executing
 ```
 
-Is to make sure that a job can be scheduled any number of times but only executed a single time per argument provided to the job we call this runtime uniqueness. This is probably most useful forbackground jobs that are fast to execute. (See mhenrixon/sidekiq-unique-jobs#111 for a great example of when this would be right.) While the job is executing/performing no other jobs can be executed at the same time.
+Is to make sure that a job can be scheduled any number of times but only executed a single time per argument provided to the job we call this runtime uniqueness. This is probably most useful for background jobs that are fast to execute. (See mhenrixon/sidekiq-unique-jobs#111 for a great example of when this would be right.) While the job is executing/performing no other jobs can be executed at the same time.
 
 ### Until Executing
 
@@ -212,13 +212,16 @@ Start the console with the following command `bundle exec jobs console`.
 
 ## Communication
 
-There is a [![Join the chat at https://gitter.im/mhenrixon/sidekiq-unique-jobs](https://badges.gitter.im/mhenrixon/sidekiq-unique-jobs.svg)](https://gitter.im/mhenri
-xon/sidekiq-unique-jobs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) for praise or scorn. This would be a good place to have lengthy discuss or brilliant suggestions or simply just nudge me if I forget about anything.
+There is a [![Join the chat at https://gitter.im/mhenrixon/sidekiq-unique-jobs](https://badges.gitter.im/mhenrixon/sidekiq-unique-jobs.svg)](https://gitter.im/mhenrixon/sidekiq-unique-jobs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) for praise or scorn. This would be a good place to have lengthy discuss or brilliant suggestions or simply just nudge me if I forget about anything.
 
 ## Testing
 
 To enable the testing for `sidekiq-unique-jobs`, add `require 'sidekiq_unique_jobs/testing'` to your testing helper.
 
+You can if you want use `gem 'mock_redis'` to prevent sidekiq unique jobs using redis.
+
+See https://github.com/mhenrixon/sidekiq-unique-jobs/tree/master/rails_example/spec/controllers/work_controller_spec.rb for an example of how to configure sidekiq and unique jobs without redis.
+
 ## Contributing
 
 1. Fork it

data/circle.yml

@@ -1,7 +1,7 @@
 machine:
   ruby:
     version:
-      2.3.0
+      2.2.3
   java:
     version: oraclejdk8
   environment:
@@ -14,21 +14,19 @@ dependencies:
     - rvm install rbx-2
     - rvm install 2.0.0
     - rvm install 2.1.7
-    - rvm install 2.2.3
   override:
     - rvm-exec jruby-9.0.0.0 bundle install
     - rvm-exec jruby-1.7 bundle install
     - rvm-exec rbx-2 bundle install
     - rvm-exec 2.0.0 bundle install
-    - rvm-exec 2.1.8 bundle install
-    - rvm-exec 2.2.4 bundle install
+    - rvm-exec 2.1.7 bundle install
+    - rvm-exec 2.2.3 bundle install
     - rvm-exec jruby-9.0.0.0 bundle exec appraisal install
     - rvm-exec jruby-1.7 bundle exec appraisal install
     - rvm-exec rbx-2 bundle exec appraisal install
     - rvm-exec 2.0.0 bundle exec appraisal install
     - rvm-exec 2.1.7 bundle exec appraisal install
     - rvm-exec 2.2.3 bundle exec appraisal install
-    - rvm-exec 2.3.0 bundle exec appraisal install
 
 test:
   override:
@@ -38,4 +36,3 @@ test:
     - rvm-exec 2.0.0 bundle exec appraisal rspec
     - rvm-exec 2.1.7 bundle exec appraisal rspec
     - rvm-exec 2.2.3 bundle exec appraisal rspec
-    - rvm-exec 2.3.0 bundle exec appraisal rspec

data/lib/sidekiq-unique-jobs.rb

@@ -57,6 +57,10 @@ module SidekiqUniqueJobs
     worker_class
   end
 
+  def mocked?
+    config.redis_test_mode == :mock
+  end
+
   def redis_version
     @redis_version ||= connection { |c| c.info('server'.freeze)['redis_version'.freeze] }
   end

data/lib/sidekiq_unique_jobs/cli.rb

@@ -12,10 +12,10 @@ module SidekiqUniqueJobs
     option :dry_run, aliases: :d, type: :boolean, desc: 'set to false to perform deletion'
     option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
     def del(pattern)
-      Util.del(pattern, count, dry_run)
+      Util.del(pattern, options[:count], options[:dry_run])
     end
 
-    desc 'drop into a console', 'easy access to helper methods'
+    desc 'console', 'drop into a console with easy access to helper methods'
     def console
       puts "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
       puts "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -20,7 +20,11 @@ module SidekiqUniqueJobs
         operative = false
         raise
       ensure
-        callback.call if operative && unlock(:server)
+        if operative && unlock(:server)
+          callback.call
+        else
+          logger.fatal { "the unique_key: #{unique_key} needs to be unlocked manually" }
+        end
       end
 
       def unlock(scope)

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -17,7 +17,9 @@ module SidekiqUniqueJobs
         sleep 0.001 until locked?
 
         yield
-
+      rescue Sidekiq::Shutdown
+        logger.fatal { "the unique_key: #{@unique_digest} needs to be unlocked manually" }
+        raise
       ensure
         SidekiqUniqueJobs.connection(@redis_pool) { |c| c.del @unique_digest }
         @mutex.unlock

data/lib/sidekiq_unique_jobs/script_mock.rb

@@ -0,0 +1,68 @@
+require 'pathname'
+require 'digest/sha1'
+
+module SidekiqUniqueJobs
+  module ScriptMock
+    module_function
+
+    extend SingleForwardable
+    def_delegator :SidekiqUniqueJobs, :connection
+
+    def call(file_name, redis_pool, options = {})
+      send(file_name, redis_pool, options)
+    end
+
+    def acquire_lock(redis_pool, options = {})
+      connection(redis_pool) do |redis|
+        unique_key = options[:keys][0]
+        job_id     = options[:argv][0]
+        expires    = options[:argv][1].to_i
+        stored_jid = redis.get(unique_key)
+
+        return stored_jid == job_id ? 1 : 0 if stored_jid
+
+        if redis.set(unique_key, job_id, nx: true, ex: expires)
+          redis.hsetnx('uniquejobs', job_id, unique_key)
+          return 1
+        else
+          return 0
+        end
+      end
+    end
+
+    def release_lock(redis_pool, options = {})
+      connection(redis_pool) do |redis|
+        unique_key = options[:keys][0]
+        job_id     = options[:argv][0]
+        stored_jid = redis.get(unique_key)
+
+        return -1 unless stored_jid
+        return 0 unless stored_jid == job_id || stored_jid == '2'
+
+        redis.del(unique_key)
+        return 1
+      end
+    end
+
+    def synchronize(redis_pool, options = {})
+      connection(redis_pool) do |redis|
+        unique_key = options[:keys][0]
+        time       = options[:argv][0].to_i
+        expires    = options[:argv][1].to_f
+
+        stored_jid = redis.get(unique_key).to_i
+        return 1 if redis.set(unique_key, time + expires, nx: true, ex: expires)
+
+        stored_time = redis.get(unique_key)
+        if stored_time && stored_time < time
+          if redis.set(unique_key, time + expires, xx: true, ex: expires)
+            return 1
+          end
+        end
+
+        return 0
+      end
+    end
+  end
+  # rubocop:enable MethodLength
+end

data/lib/sidekiq_unique_jobs/scripts.rb

@@ -28,11 +28,14 @@ module SidekiqUniqueJobs
         redis.evalsha(script_shas[file_name], options)
       end
     rescue Redis::CommandError => ex
-      raise ScriptError,
-            "#{file_name}.lua\n\n" +
-            ex.message + "\n\n" +
-            script_source(file_name) +
-            ex.backtrace.join("\n")
+      if ex.message == 'NOSCRIPT No matching script. Please use EVAL.'
+        script_shas[file_name] = nil
+        call(file_name, redis_pool, options)
+      else
+        raise ScriptError,
+              "#{file_name}.lua\n\n#{ex.message}\n\n#{script_source(file_name)}" \
+              "#{ex.backtrace.join("\n")}"
+      end
     end
 
     def script_source(file_name)

data/lib/sidekiq_unique_jobs/testing.rb

@@ -1,6 +1,42 @@
 require 'sidekiq_unique_jobs/testing/sidekiq_overrides'
+require 'sidekiq_unique_jobs/script_mock'
 
 module SidekiqUniqueJobs
+  alias redis_version_real redis_version
+  def redis_version
+    if mocked?
+      '0.0'
+    else
+      redis_version_real
+    end
+  end
+
+  module Scripts
+    module Overrides
+      def self.included(base)
+        base.extend Testing
+        base.class_eval do
+          class << self
+            alias_method :call_orig, :call
+            alias_method :call, :call_ext
+          end
+        end
+      end
+
+      module Testing
+        def call_ext(file_name, redis_pool, options = {})
+          if SidekiqUniqueJobs.config.redis_test_mode == :mock
+            SidekiqUniqueJobs::ScriptMock.call(file_name, redis_pool, options)
+          else
+            call_orig(file_name, redis_pool, options)
+          end
+        end
+      end
+    end
+
+    include Overrides
+  end
+
   module Client
     class Middleware
       alias call_real call

data/lib/sidekiq_unique_jobs/testing/sidekiq_overrides.rb

@@ -59,7 +59,7 @@ module Sidekiq
 
       module Testing
         def clear_all_ext
-          SidekiqUniqueJobs::Util.del('*', 1000, false)
+          SidekiqUniqueJobs::Util.del('*', 1000, false) unless SidekiqUniqueJobs.mocked?
           clear_all_orig
         end
       end

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,3 +1,3 @@
 module SidekiqUniqueJobs
-  VERSION = '4.0.17'.freeze
+  VERSION = '4.0.18'.freeze
 end

data/rails_example/.env

@@ -0,0 +1,12 @@
+SECRET_KEY_BASE=10167c7f7654ed02b3557b05b88ece1866a9748a2e8e064016f76cb74ddae38264f7403ba65f301d63c21d770e0ee2ac6906f245e904b3b140d12297f67e0db1
+APP_DOMAIN=localhost
+APP_WEB_URL=http://localhost:3000
+APP_CABLE_URL=ws://localhost:28080
+WEB_CONSOLE_WHITELISTED_IPS=127.0.0.1 ::1 127.0.0.0/8 ::1
+DB_HOST=postgres
+DB_PORT=5432
+DB_USERNAME=mhenrixon
+DB_PASSWORD=
+DB_POOL=25
+DB_TIMEOUT=5000
+DB_REAPING_FREQUENCY=10

data/rails_example/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/rails_example/Gemfile

@@ -1,20 +1,34 @@
 source 'https://rubygems.org'
 
-ruby '2.2.3'
+ruby '2.3.1'
+
+gem 'rails', '~> 5.0'
+gem 'puma'
+gem 'json'
+gem 'bigdecimal'
 
 platforms :ruby do
-  gem 'sqlite3'
+  gem 'pg'
   gem 'pry-byebug'
   gem 'ruby-prof'
 end
 
 platforms :jruby do
   gem 'jruby-openssl'
-  gem 'activerecord-jdbcsqlite3-adapter'
+  gem 'jdbc-pg'
+  gem 'activerecord-jdbc-adapter'
 end
 
-gem 'rails'
-gem 'sidekiq-unique-jobs', path: '..'
-gem 'sinatra'
-gem 'web-console', '~> 2.0'
+gem 'sidekiq-unique-jobs', path: '../'
+gem 'sinatra', github: 'sinatra/sinatra'
+
+group :development do
+  gem 'web-console'
+end
+
+group :development, :test do
+  gem 'rspec-rails'
+  gem 'factory_girl_rails'
+  gem 'mock_redis'
+end
 

data/rails_example/app/channels/appearance_channel.rb

@@ -0,0 +1,17 @@
+class AppearanceChannel < ApplicationCable::Channel
+  def subscribed
+    current_user.appear
+  end
+
+  def unsubscribed
+    current_user.disappear
+  end
+
+  def appear(data)
+    current_user.appear on: data['appearing_on']
+  end
+
+  def away
+    current_user.away
+  end
+end

data/rails_example/app/channels/application_cable/channel.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Channel < ActionCable::Channel::Base
+  end
+end

data/rails_example/app/channels/application_cable/connection.rb

@@ -0,0 +1,9 @@
+module ApplicationCable
+  class Connection < ActionCable::Connection::Base
+    identified_by :current_user
+
+    def connect
+      self.current_user = Guest.new
+    end
+  end
+end

data/rails_example/app/channels/post_channel.rb

@@ -0,0 +1,5 @@
+class PostChannel < ApplicationCable::Channel
+  def subscribed
+    stream_from "post_#{params[:post_id]}"
+  end
+end

data/rails_example/app/controllers/work_controller.rb

@@ -1,9 +1,12 @@
 class WorkController < ApplicationController
-  def duplicate
-    SimpleWorker.perform_async(1)
-    4.times do |_x|
-      SpawnSimpleWorker.perform_async(1)
-    end
+  def duplicate_simple
+    4.times { SimpleWorker.perform_async(1) }
+
+    redirect_to '/sidekiq'
+  end
+
+  def duplicate_nested
+    4.times { SpawnSimpleWorker.perform_async(1) }
 
     redirect_to '/sidekiq'
   end