sidekiq-unique-jobs 4.0.18 → 5.0.0

data/lib/sidekiq_unique_jobs/scripts.rb

@@ -1,5 +1,6 @@
 require 'pathname'
 require 'digest/sha1'
+require 'concurrent/map'
 
 module SidekiqUniqueJobs
   ScriptError = Class.new(StandardError)
@@ -8,29 +9,29 @@ module SidekiqUniqueJobs
     LUA_PATHNAME ||= Pathname.new(__FILE__).dirname.join('../../redis').freeze
     SOURCE_FILES ||= Dir[LUA_PATHNAME.join('**/*.lua')].compact.freeze
     DEFINED_METHODS ||= [].freeze
+    SCRIPT_SHAS ||= Concurrent::Map.new
 
     module_function
 
     extend SingleForwardable
     def_delegator :SidekiqUniqueJobs, :connection
 
-    def script_shas
-      @script_shas ||= {}
-    end
-
     def logger
       Sidekiq.logger
     end
 
-    def call(file_name, redis_pool, options = {})
+    def call(file_name, redis_pool, options = {}) # rubocop:disable MethodLength
       connection(redis_pool) do |redis|
-        script_shas[file_name] ||= redis.script(:load, script_source(file_name))
-        redis.evalsha(script_shas[file_name], options)
+        if SCRIPT_SHAS[file_name].nil?
+          SCRIPT_SHAS[file_name] = redis.script(:load, script_source(file_name))
+        end
+        redis.evalsha(SCRIPT_SHAS[file_name], options)
       end
     rescue Redis::CommandError => ex
       if ex.message == 'NOSCRIPT No matching script. Please use EVAL.'
-        script_shas[file_name] = nil
+        SCRIPT_SHAS.delete(file_name)
         call(file_name, redis_pool, options)
+        raise
       else
         raise ScriptError,
               "#{file_name}.lua\n\n#{ex.message}\n\n#{script_source(file_name)}" \

data/lib/sidekiq_unique_jobs/testing/sidekiq_overrides.rb

@@ -4,26 +4,30 @@ module Sidekiq
   module Worker
     module ClassMethods
       # Drain and run all jobs for this worker
-      def drain
-        while (job = jobs.shift)
+      unless Sidekiq::Testing.respond_to?(:server_middleware)
+        def drain
+          while (job = jobs.shift)
+            worker = new
+            worker.jid = job['jid']
+            worker.bid = job['bid'] if worker.respond_to?(:bid=)
+            execute_job(worker, job['args'])
+            unlock(job) if Sidekiq::Testing.fake?
+          end
+        end
+      end
+
+      # Pop out a single job and perform it
+      unless Sidekiq::Testing.respond_to?(:server_middleware)
+        def perform_one
+          raise(EmptyQueueError, 'perform_one called with empty job queue') if jobs.empty?
+          job = jobs.shift
           worker = new
           worker.jid = job['jid']
           worker.bid = job['bid'] if worker.respond_to?(:bid=)
           execute_job(worker, job['args'])
           unlock(job) if Sidekiq::Testing.fake?
         end
-      end unless Sidekiq::Testing.respond_to?(:server_middleware)
-
-      # Pop out a single job and perform it
-      def perform_one
-        raise(EmptyQueueError, 'perform_one called with empty job queue') if jobs.empty?
-        job = jobs.shift
-        worker = new
-        worker.jid = job['jid']
-        worker.bid = job['bid'] if worker.respond_to?(:bid=)
-        execute_job(worker, job['args'])
-        unlock(job) if Sidekiq::Testing.fake?
-      end unless Sidekiq::Testing.respond_to?(:server_middleware)
+      end
 
       # Clear all jobs for this worker
       def clear
@@ -37,9 +41,11 @@ module Sidekiq
         # end
       end
 
-      def execute_job(worker, args)
-        worker.perform(*args)
-      end unless respond_to?(:execute_job)
+      unless respond_to?(:execute_job)
+        def execute_job(worker, args)
+          worker.perform(*args)
+        end
+      end
 
       def unlock(job)
         SidekiqUniqueJobs::Unlockable.unlock(job)

data/lib/sidekiq_unique_jobs/unique_args.rb

@@ -22,7 +22,7 @@ module SidekiqUniqueJobs
         @worker_class ||= worker_class_constantize(@item[CLASS_KEY])
         @item[UNIQUE_PREFIX_KEY] ||= unique_prefix
         @item[UNIQUE_ARGS_KEY] = unique_args(@item[ARGS_KEY]) # SIC! Calculate unique_args unconditionally
-        @item[UNIQUE_DIGEST_KEY] ||= unique_digest
+        @item[UNIQUE_DIGEST_KEY] = unique_digest
       end
     end
 

data/lib/sidekiq_unique_jobs/unlockable.rb

@@ -7,24 +7,13 @@ module SidekiqUniqueJobs
     end
 
     def unlock_by_key(unique_key, jid, redis_pool = nil)
-      Scripts.call(:release_lock, redis_pool, keys: [unique_key], argv: [jid]) do |result|
-        after_unlock(result, __method__)
-      end
-    end
-
-    def unlock_by_jid(jid, redis_pool = nil)
-      Scripts.call(:release_lock_by_jid, redis_pool, keys: [jid]) do |result|
-        after_unlock(result, __method__)
-      end
+      result = Scripts.call(:release_lock, redis_pool, keys: [unique_key], argv: [jid])
+      after_unlock(result, __method__, unique_key, jid)
     end
 
-    def unlock_by_arguments(_worker_class, _unique_arguments = {})
-      Scripts.call(:release_lock, redis_pool, keys: [unique_key], argv: [jid]) do |result|
-        after_unlock(result, __method__)
-      end
-    end
+    def after_unlock(result, calling_method, unique_key, jid) # rubocop:disable Metrics/MethodLength
+      ensure_job_id_removed(jid)
 
-    def after_unlock(result, calling_method)
       case result
       when 1
         logger.debug { "successfully unlocked #{unique_key}" }
@@ -40,6 +29,10 @@ module SidekiqUniqueJobs
       end
     end
 
+    def ensure_job_id_removed(jid)
+      Sidekiq.redis { |redis| redis.hdel(SidekiqUniqueJobs::HASH_KEY, jid) }
+    end
+
     def logger
       Sidekiq.logger
     end

data/lib/sidekiq_unique_jobs/util.rb

@@ -11,7 +11,14 @@ module SidekiqUniqueJobs
       send("keys_by_#{redis_keys_method}", pattern, count)
     end
 
+    def unique_key(jid)
+      connection do |redis|
+        redis.hget(SidekiqUniqueJobs::HASH_KEY, jid)
+      end
+    end
+
     def del(pattern = SCAN_PATTERN, count = 0, dry_run = true)
+      raise 'Please provide a number of keys to delete greater than zero' if count.zero?
       logger.debug { "Deleting keys by: #{pattern}" }
       keys, time = timed { keys(pattern, count) }
       logger.debug { "#{keys.size} matching keys found in #{time} sec." }

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,3 +1,3 @@
 module SidekiqUniqueJobs
-  VERSION = '4.0.18'.freeze
+  VERSION = '5.0.0'.freeze
 end

data/rails_example/.env.test

@@ -0,0 +1,12 @@
+SECRET_KEY_BASE=10167c7f7654ed02b3557b05b88ece1866a9748a2e8e064016f76cb74ddae38264f7403ba65f301d63c21d770e0ee2ac6906f245e904b3b140d12297f67e0db1
+APP_DOMAIN=localhost
+APP_WEB_URL=http://localhost:3000
+APP_CABLE_URL=ws://localhost:28080
+WEB_CONSOLE_WHITELISTED_IPS=127.0.0.1 ::1 127.0.0.0/8 ::1
+DB_HOST=localhost
+DB_PORT=5432
+DB_USERNAME=postgres
+DB_PASSWORD=3x4mpl3
+DB_POOL=25
+DB_TIMEOUT=5000
+DB_REAPING_FREQUENCY=10

data/rails_example/.rspec

@@ -1,2 +1,2 @@
 --color
---require spec_helper
+--order random

data/rails_example/Gemfile

@@ -1,11 +1,11 @@
-source 'https://rubygems.org'
+source 'https://rubygems.org' # rubocop:disable Style/FileName
 
-ruby '2.3.1'
+ruby '2.4.1'
 
-gem 'rails', '~> 5.0'
-gem 'puma'
-gem 'json'
 gem 'bigdecimal'
+gem 'json'
+gem 'puma'
+gem 'rails', '~> 5.0'
 
 platforms :ruby do
   gem 'pg'
@@ -14,11 +14,13 @@ platforms :ruby do
 end
 
 platforms :jruby do
-  gem 'jruby-openssl'
-  gem 'jdbc-pg'
   gem 'activerecord-jdbc-adapter'
+  gem 'jdbc-pg'
+  gem 'jruby-openssl'
 end
 
+# gem 'sidekiq-unique-jobs'
+gem 'rack-protection' # , github: 'sinatra/rack-protection'
 gem 'sidekiq-unique-jobs', path: '../'
 gem 'sinatra', github: 'sinatra/sinatra'
 
@@ -27,8 +29,9 @@ group :development do
 end
 
 group :development, :test do
-  gem 'rspec-rails'
+  gem 'dockerspec'
   gem 'factory_girl_rails'
+  gem 'fuubar'
   gem 'mock_redis'
+  gem 'rspec-rails'
 end
-

data/rails_example/app/controllers/work_controller.rb

@@ -1,13 +1,27 @@
 class WorkController < ApplicationController
   def duplicate_simple
-    4.times { SimpleWorker.perform_async(1) }
+    4.times { SimpleWorker.perform_async(unique_argument) }
+
+    redirect_to '/sidekiq'
+  end
+
+  def duplicate_slow
+    4.times { SlowUntilExecutingWorker.perform_async(unique_argument) }
 
     redirect_to '/sidekiq'
   end
 
   def duplicate_nested
-    4.times { SpawnSimpleWorker.perform_async(1) }
+    4.times { SpawnSimpleWorker.perform_async(unique_argument) }
 
     redirect_to '/sidekiq'
   end
+
+  def unique_argument
+    params[:id]
+  end
+
+  def safe_params
+    params.permit!(:id)
+  end
 end

data/rails_example/app/models/guest.rb

@@ -1,21 +1,18 @@
 class Guest
+  GUEST_NAME = 'Guest Visitor'.freeze
+  GUEST_EMAIL = 'unknown@domain.com'.freeze
+
   def name
-    'Guest Visitor'.freeze
+    GUEST_NAME
   end
 
   def email
-    'unknown@domain.com'.freeze
-  end
-
-  def appear
+    GUEST_EMAIL
   end
 
-  def disappear
-  end
+  def appear; end
 
-  def appear
-  end
+  def disappear; end
 
-  def away
-  end
+  def away; end
 end

data/rails_example/app/workers/simple_worker.rb

@@ -6,7 +6,10 @@ class SimpleWorker
                     [args.first]
                   end)
 
-  def perform(_some_args)
+  def perform(some_args)
+    Sidekiq::Logging.with_context(self.class.name) do
+      Sidekiq.logger.debug { "#{__method__}(#{some_args})" }
+    end
     sleep 1
   end
 end

data/rails_example/app/workers/slow_until_executing_worker.rb

@@ -0,0 +1,15 @@
+class SlowUntilExecutingWorker
+  include Sidekiq::Worker
+  sidekiq_options unique: :until_executing,
+                  queue: :default,
+                  unique_args: (lambda do |args|
+                    [args.first]
+                  end)
+
+  def perform(some_args)
+    Sidekiq::Logging.with_context(self.class.name) do
+      Sidekiq.logger.debug { "#{__method__}(#{some_args})" }
+    end
+    sleep 15
+  end
+end

data/rails_example/app/workers/spawn_simple_worker.rb

@@ -2,6 +2,9 @@ class SpawnSimpleWorker
   include Sidekiq::Worker
 
   def perform(spawn_arg)
+    Sidekiq::Logging.with_context(self.class.name) do
+      Sidekiq.logger.debug { "#{__method__}(#{spawn_arg})" }
+    end
     SimpleWorker.perform_async spawn_arg
   end
 end

data/rails_example/app/workers/without_argument_worker.rb

@@ -0,0 +1,12 @@
+class WithoutArgumentWorker
+  include Sidekiq::Worker
+  sidekiq_options unique: :until_executed,
+                  log_duplicate_payload: true
+
+  def perform
+    Sidekiq::Logging.with_context(self.class.name) do
+      Sidekiq.logger.debug { __method__.to_s }
+    end
+    sleep 20
+  end
+end

data/rails_example/bin/check_or_setup_db

@@ -28,7 +28,6 @@ rescue PG::ConnectionBad
   end
   1
 rescue ActiveRecord::NoDatabaseError, ActiveRecord::AdapterNotSpecified
-
   include ActiveRecord::Tasks
 
   DatabaseTasks.root = File.expand_path '../..', __FILE__

data/rails_example/common-services.yml

@@ -25,6 +25,8 @@ services:
       # to our redis container:
       REDIS_URL: redis://redis:6379
 
+      DATABASE_URL: postgresql://postgres:3x4mpl3@postgres:5432/rails_example_dev
+
       # We'll set the RAILS_ENV and RACK_ENV environment variables to
       # 'development', so our app containers will start in 'development' mode
       # on this compose project:

data/rails_example/config/database.yml

@@ -3,8 +3,8 @@ defaults: &defaults
   adapter: postgresql
   host: <%= ENV.fetch('DB_HOST') { 'localhost' } %>
   port: <%= ENV.fetch('DB_PORT') { 5432 } %>
-  username: <%= ENV.fetch('DB_USERNAME') { 'postgres' } %>
-  password: <%= ENV.fetch('DB_PASSWORD') { '' } %>
+  username: <%= ENV.fetch('DB_USERNAME') { 'mhenrixon' } %>
+  password: <%= ENV.fetch('DB_PASSWORD') { nil } %>
   pool: <%= ENV.fetch('DB_POOL') { 25 } %>
   timeout: <%= ENV.fetch('DB_TIMEOUT') { 5000 } %>
   reaping_frequency: <%= ENV.fetch('DB_REAPING_FREQUENCY') { 5000 } %>

data/rails_example/config/environments/development.rb

@@ -18,7 +18,7 @@ Rails.application.configure do
 
     config.cache_store = :memory_store
     config.public_file_server.headers = {
-      'Cache-Control' => 'public, max-age=172800'
+      'Cache-Control' => 'public, max-age=172800',
     }
   else
     config.action_controller.perform_caching = false

data/rails_example/config/environments/test.rb

@@ -15,7 +15,7 @@ Rails.application.configure do
   # Configure public file server for tests with Cache-Control for performance.
   config.public_file_server.enabled = true
   config.public_file_server.headers = {
-    'Cache-Control' => 'public, max-age=3600'
+    'Cache-Control' => 'public, max-age=3600',
   }
 
   # Show full error reports and disable caching.

data/rails_example/config/initializers/sidekiq.rb

@@ -1,6 +1,6 @@
 Sidekiq.default_worker_options = {
   backtrace: true,
-  retry: false
+  retry: false,
 }
 
 Sidekiq.configure_server do |config|

data/rails_example/config/routes.rb

@@ -3,4 +3,5 @@ Rails.application.routes.draw do
   mount Sidekiq::Web, at: '/sidekiq'
   get 'work/duplicate_simple' => 'work#duplicate_simple'
   get 'work/duplicate_nested' => 'work#duplicate_nested'
+  get 'work/duplicate_slow' => 'work#duplicate_slow'
 end

data/rails_example/config/spring.rb

@@ -1,6 +1,6 @@
-%w(
+%w[
   .ruby-version
   .rbenv-vars
   tmp/restart.txt
   tmp/caching-dev.txt
-).each { |path| Spring.watch(path) }
+].each { |path| Spring.watch(path) }

data/rails_example/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2'
+version: '3.1'
 volumes:
   postgres-data:
     driver: local
@@ -14,50 +14,31 @@ networks:
     driver: bridge
 
 services:
-  jobs: &app
-    extends:
-      file: common-services.yml
-      service: rails
+  rspec:
+    image: ruby:2.4
+    working_dir: /usr/src/app
+    stdin_open: true
+    tty: true
+    volumes:
+      - .:/usr/src/app
+      - gems:/usr/local/bundle
     entrypoint: /usr/src/app/dev-entrypoint.sh
-    command: bundle exec sidekiq -c 25
+    command: bundle exec rspec
     networks:
       - back-tier
-    depends_on:
-      - postgres
-      - redis
-
-  web:
-    extends:
-      file: common-services.yml
-      service: rails
-    entrypoint: /usr/src/app/dev-entrypoint.sh
-    command: bundle exec puma --bind tcp://0.0.0.0:3000
-    ports:
-      - 3000:3000
-    networks:
       - front-tier
-      - back-tier
-    depends_on:
-      - postgres
-      - redis
-
-  cable:
-    extends:
-      file: common-services.yml
-      service: rails
-    entrypoint: /usr/src/app/dev-entrypoint.sh
-    command: bundle exec puma --bind tcp://0.0.0.0:28080 cable.ru
-    ports:
-      - 28080:28080
-    networks:
-      - front-tier
-      - back-tier
     depends_on:
       - postgres
       - redis
+    environment:
+      PATH: /usr/src/app/bin:/usr/local/bundle/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+      REDIS_URL: redis://redis:6379
+      DATABASE_URL: postgresql://postgres:3x4mpl3@postgres:5433/rails_example_test
+      RAILS_ENV: test
+      RACK_ENV: test
 
   redis:
-    image: redis
+    image: redis:latest
     ports:
       # We'll bind our host's port 6379 to redis's port 6379, so we can use
       # Redis Desktop Manager (or other tools) with it:
@@ -72,11 +53,11 @@ services:
 
 
   postgres:
-    image: postgres
+    image: postgres:latest
     ports:
       # We'll bind our host's port 5432 to postgres's port 5432, so we can use
             # our database IDEs with it:
-      - 5432:5432
+      - 5433:5433
     volumes:
       # We'll store the postgres data in the 'postgres-data' volume we defined:
       - postgres-data:/var/lib/postgresql/data