sidekiq-unique-jobs 4.0.8 → 4.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64a81ae8142ce1f78d40061b8c1fbe11cd9ec79a
-  data.tar.gz: 6fe4d1d19eaa60088d3a7a46ee9761b7de1cfecb
+  metadata.gz: 10b0a3b898db3bbf29feae82657b62d4dfa5e248
+  data.tar.gz: b3a13fd8edcc4227478a3927bfa7a005d1bac46b
 SHA512:
-  metadata.gz: 7b0044b2721c29fa5ffbb342c670c2d8f087719c12e7ad59dc9a30e25b9d9a1ba54bd3cdbbe738202bc4f39a1c27835a1e964837ccdec57fc01c43c2ed0efd6e
-  data.tar.gz: 2429e2b73e64cbbdb2972278309ca45ecb1dccff56777c65f15dacd05162c59c755d7a55a1c93cbbc71c13bcc4bfdb71a03496dda881842ddc0b5a3205237008
+  metadata.gz: e7c316aced14d335b51142a5c1f0270f3231458edac5545dda7e95252209f7d45408140d380f071505e0ffce9314ab82aed697064fc9a1d39eccea5925502333
+  data.tar.gz: f8243f502b22d28a56f7a6c102dee27dc460dbd38dd82b53236759dc3a81b577ca95196be4e4de60f6676c358fea323385f8128b2443779fd5c2b85cf8f4d137

data/.fasterer.yml

@@ -0,0 +1,23 @@
+speedups:
+  parallel_assignment: false
+  rescue_vs_respond_to: true
+  module_eval: true
+  shuffle_first_vs_sample: true
+  for_loop_vs_each: true
+  each_with_index_vs_while: false
+  map_flatten_vs_flat_map: true
+  reverse_each_vs_reverse_each: true
+  select_first_vs_detect: true
+  sort_vs_sort_by: true
+  fetch_with_argument_vs_block: true
+  keys_each_vs_each_key: true
+  hash_merge_bang_vs_hash_brackets: true
+  block_vs_symbol_to_proc: true
+  proc_call_vs_yield: true
+  gsub_vs_tr: true
+  select_last_vs_reverse_detect: true
+  getter_vs_attr_reader: true
+  setter_vs_attr_writer: true
+
+exclude_paths:
+  - 'spec/**/*.rb'

data/.rubocop.yml

@@ -2,9 +2,10 @@ AllCops:
   Include:
     - '**/Rakefile'
     - '**/lib'
+    - '**/bin'
     - '**/spec'
   Exclude:
-    - 'bin/**/*'
+    - 'Gemfile.lock'
     - 'gemfiles/**/*'
 
 Lint/HandleExceptions:

data/CHANGELOG.md

@@ -1,6 +1,17 @@
+## v4.0.9
+
+- Add command line and console extensions for removal of unique jobs (c292d87)
+
+## v4.0.8
+
+- Use unique arguments for the `WhileExecuting` lock (#127)
+- Delicensed code (#132)
+- Fix queuing unique jobs (#138)
+
 ## v4.0.7
 
 - Use unique arguments for the `WhileExecuting` lock (#127)
+- See also https://github.com/mhenrixon/sidekiq-unique-jobs/releases/tag/v4.0.7
 
 ## v4.0.6
 
@@ -47,7 +58,7 @@
 
 - Improve uniqueness check performance thanks @mpherham
 - Remove locks in sidekiq fake testing mode
-- Do not unlock jobs when sidekiq is shutting down 
+- Do not unlock jobs when sidekiq is shutting down
 
 ## v3.0.13
 

data/Gemfile

@@ -2,11 +2,12 @@ source 'http://rubygems.org'
 gemspec
 
 gem 'appraisal', '~> 2.0.0'
-gem 'rspec-its',     require: false
+gem 'rspec-its',  require: false
+gem 'rspec-wait', require: false
 
 platform :mri_22 do
-  gem 'pry',                       require: false
-  gem 'pry-rescue',                require: false
+  gem 'fasterer',                  require: false
+  gem 'benchmark-ips',             require: false
   gem 'pry-byebug',                require: false
   gem 'simplecov-json',            require: false
   gem 'memory_profiler',           require: false

data/README.md

@@ -28,11 +28,11 @@ Or install it yourself as:
 
     $ gem install sidekiq-unique-jobs
 
-## A word on locking
+## Locking
 
-Like @mperham mentions on [this wiki page](https://github.com/mperham/sidekiq/wiki/Related-Projects#unique-jobs) it is hard to enforce uniqueness with redis in a distributed redis setting. 
+Like @mperham mentions on [this wiki page](https://github.com/mperham/sidekiq/wiki/Related-Projects#unique-jobs) it is hard to enforce uniqueness with redis in a distributed redis setting.
 
-To make things worse there are many ways of wanting to enforce uniqueness. 
+To make things worse there are many ways of wanting to enforce uniqueness.
 
 ### While Executing
 
@@ -74,7 +74,7 @@ The job won't be unlocked until the timeout/expiry runs out.
 sidekiq_options unique: :until_and_while_executing
 ```
 
-This lock is exactly what you would expect. It is considered unique in a way until executing begins and it is locked while executing so what differs from `UntilExecuted`? 
+This lock is exactly what you would expect. It is considered unique in a way until executing begins and it is locked while executing so what differs from `UntilExecuted`?
 
 The difference is that this job has two types of uniqueness:
 1. It is unique until execution
@@ -187,7 +187,24 @@ class UniqueJobWithFilterMethod
 end
 ```
 
-### Testing
+## Debugging
+There are two ways to display and remove keys regarding uniqueness. The console way and the command line way.
+
+### Console
+Start the console with the following command `bundle exec jobs console`.
+
+#### List Unique Keys
+`keys '*', count: 100`
+
+#### Remove Unique Keys
+`del_by '*', count: 100, dry_run: false` the dry_run and count parameters are both required. This is to have some type of protection against clearing out all uniqueness.
+
+### Command Line
+
+`bundle exec jobs` displays help on how to use the unique jobs command line.
+
+
+## Testing
 
 To enable the testing for `sidekiq-unique-jobs`, add `require 'sidekiq_unique_jobs/testing'` to your testing helper.
 

data/bin/jobs

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'sidekiq-unique-jobs'
+
+SidekiqUniqueJobs::Cli.start(ARGV)

data/gemfiles/sidekiq_2.15.gemfile

@@ -1,9 +1,9 @@
 # This file was generated by Appraisal
 
-source "http://rubygems.org"
+source 'http://rubygems.org'
 
-gem "appraisal", "~> 1.0.0"
-gem "pry", :platform => :mri
-gem "sidekiq", "2.15"
+gem 'appraisal', '~> 1.0.0'
+gem 'pry', platform: :mri
+gem 'sidekiq', '2.15'
 
-gemspec :path => "../"
+gemspec path: '../'

data/gemfiles/sidekiq_2.16.gemfile

@@ -1,9 +1,9 @@
 # This file was generated by Appraisal
 
-source "http://rubygems.org"
+source 'http://rubygems.org'
 
-gem "appraisal", "~> 1.0.0"
-gem "pry", :platform => :mri
-gem "sidekiq", "2.16"
+gem 'appraisal', '~> 1.0.0'
+gem 'pry', platform: :mri
+gem 'sidekiq', '2.16'
 
-gemspec :path => "../"
+gemspec path: '../'

data/gemfiles/sidekiq_2.17.gemfile

@@ -4,11 +4,12 @@ source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
 gem "rspec-its", :require => false
+gem "rspec-wait", :require => false
 gem "sidekiq", "~> 2.17.4"
 
 platforms :mri_22 do
-  gem "pry", :require => false
-  gem "pry-rescue", :require => false
+  gem "fasterer", :require => false
+  gem "benchmark-ips", :require => false
   gem "pry-byebug", :require => false
   gem "simplecov-json", :require => false
   gem "memory_profiler", :require => false

data/gemfiles/sidekiq_3.0.gemfile

@@ -4,11 +4,12 @@ source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
 gem "rspec-its", :require => false
+gem "rspec-wait", :require => false
 gem "sidekiq", "~> 3.0.0"
 
 platforms :mri_22 do
-  gem "pry", :require => false
-  gem "pry-rescue", :require => false
+  gem "fasterer", :require => false
+  gem "benchmark-ips", :require => false
   gem "pry-byebug", :require => false
   gem "simplecov-json", :require => false
   gem "memory_profiler", :require => false

data/gemfiles/sidekiq_3.1.gemfile

@@ -4,11 +4,12 @@ source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
 gem "rspec-its", :require => false
+gem "rspec-wait", :require => false
 gem "sidekiq", "~> 3.1.0"
 
 platforms :mri_22 do
-  gem "pry", :require => false
-  gem "pry-rescue", :require => false
+  gem "fasterer", :require => false
+  gem "benchmark-ips", :require => false
   gem "pry-byebug", :require => false
   gem "simplecov-json", :require => false
   gem "memory_profiler", :require => false

data/gemfiles/sidekiq_3.2.gemfile

@@ -4,11 +4,12 @@ source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
 gem "rspec-its", :require => false
+gem "rspec-wait", :require => false
 gem "sidekiq", "~> 3.2.0"
 
 platforms :mri_22 do
-  gem "pry", :require => false
-  gem "pry-rescue", :require => false
+  gem "fasterer", :require => false
+  gem "benchmark-ips", :require => false
   gem "pry-byebug", :require => false
   gem "simplecov-json", :require => false
   gem "memory_profiler", :require => false

data/gemfiles/sidekiq_3.3.gemfile

@@ -4,11 +4,12 @@ source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
 gem "rspec-its", :require => false
+gem "rspec-wait", :require => false
 gem "sidekiq", "~> 3.3.0"
 
 platforms :mri_22 do
-  gem "pry", :require => false
-  gem "pry-rescue", :require => false
+  gem "fasterer", :require => false
+  gem "benchmark-ips", :require => false
   gem "pry-byebug", :require => false
   gem "simplecov-json", :require => false
   gem "memory_profiler", :require => false

data/gemfiles/sidekiq_develop.gemfile

@@ -4,11 +4,12 @@ source "http://rubygems.org"
 
 gem "appraisal", "~> 2.0.0"
 gem "rspec-its", :require => false
+gem "rspec-wait", :require => false
 gem "sidekiq", :github => "mperham/sidekiq"
 
 platforms :mri_22 do
-  gem "pry", :require => false
-  gem "pry-rescue", :require => false
+  gem "fasterer", :require => false
+  gem "benchmark-ips", :require => false
   gem "pry-byebug", :require => false
   gem "simplecov-json", :require => false
   gem "memory_profiler", :require => false

data/lib/sidekiq-unique-jobs.rb

@@ -1,5 +1,8 @@
 require 'yaml' if RUBY_VERSION.include?('2.0.0') # rubocop:disable FileName
+require 'sidekiq_unique_jobs/version'
 require 'sidekiq_unique_jobs/constants'
+require 'sidekiq_unique_jobs/util'
+require 'sidekiq_unique_jobs/cli'
 require 'sidekiq_unique_jobs/core_ext'
 require 'sidekiq_unique_jobs/timeout_calculator'
 require 'sidekiq_unique_jobs/options_with_fallback'
@@ -8,7 +11,6 @@ require 'sidekiq_unique_jobs/unique_args'
 require 'sidekiq_unique_jobs/unlockable'
 require 'sidekiq_unique_jobs/lock'
 require 'sidekiq_unique_jobs/middleware'
-require 'sidekiq_unique_jobs/version'
 require 'sidekiq_unique_jobs/config'
 require 'sidekiq_unique_jobs/sidekiq_unique_ext'
 
@@ -41,7 +43,7 @@ module SidekiqUniqueJobs
   end
 
   def namespace
-    @namespace ||= Sidekiq.redis { |c| c.respond_to?(:namespace) ? c.namespace : nil }
+    @namespace ||= connection { |c| c.respond_to?(:namespace) ? c.namespace : nil }
   end
 
   # Attempt to constantize a string worker_class argument, always
@@ -54,19 +56,22 @@ module SidekiqUniqueJobs
   end
 
   def redis_version
-    @redis_version ||= Sidekiq.redis { |c| c.info('server')['redis_version'] }
+    @redis_version ||= connection { |c| c.info('server')['redis_version'] }
   end
 
-  def connection(redis_pool = nil, &block)
-    return mock_redis if config.mocking?
-    redis_pool ? redis_pool.with(&block) : Sidekiq.redis(&block)
+  def connection(redis_pool = nil)
+    if redis_pool
+      redis_pool.with { |conn| yield conn }
+    else
+      Sidekiq.redis { |conn| yield conn }
+    end
   end
 
   def mock_redis
     @redis_mock ||= MockRedis.new if defined?(MockRedis)
   end
 
-  def synchronize(item, redis_pool, &blk)
-    Lock::WhileExecuting.synchronize(item, redis_pool, &blk)
+  def synchronize(item, redis_pool)
+    Lock::WhileExecuting.synchronize(item, redis_pool) { yield }
   end
 end

data/lib/sidekiq/simulator.rb

@@ -0,0 +1,75 @@
+require 'sidekiq/launcher'
+require 'timeout'
+
+module Sidekiq
+  class Simulator
+    attr_reader :queues, :launcher
+
+    def self.process_queue(queue)
+      new(queue).process_queue { yield }
+    end
+
+    def initialize(queue)
+      @queues = [queue].flatten.uniq
+      @launcher = Sidekiq::Launcher.new(sidekiq_options(queues))
+    end
+
+    def process_queue
+      run_launcher { yield }
+    ensure
+      terminate_launcher
+    end
+
+    private
+
+    def run_launcher
+      using_timeout(15) do
+        launcher.run
+        sleep 0.001 until alive?
+      end
+    rescue Timeout::Error => e
+      logger.warn { "Timeout while running #{__method__}" }
+      logger.warn { e }
+    ensure
+      yield
+    end
+
+    def terminate_launcher
+      if launcher.respond_to?(:alive?)
+        launcher.terminate # Better to be fast than graceful for our purposes
+      else
+        launcher.stop # New sidekiq works better
+      end
+    end
+
+    def alive?
+      if launcher.respond_to?(:alive?)
+        launcher.alive?
+      else
+        launcher.manager.workers.size > 0
+      end
+    end
+
+    def stopped?
+      !alive?
+    end
+
+    def using_timeout(value)
+      Timeout.timeout(value) do
+        yield
+      end
+    end
+
+    def sidekiq_options(queues = [])
+      { queues: queues,
+        concurrency: 3,
+        timeout: 3,
+        verbose: false,
+        logfile: './tmp/sidekiq.log' }
+    end
+
+    def logger
+      @logger ||= Sidekiq.logger
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/cli.rb

@@ -0,0 +1,34 @@
+require 'thor'
+
+module SidekiqUniqueJobs
+  class Cli < Thor
+    desc 'keys PATTERN', 'list all unique keys and their expiry time'
+    option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
+    def keys(pattern)
+      Util.keys(pattern, count: options[:count])
+    end
+
+    desc 'del_by PATTERN', 'deletes unique keys from redis by pattern'
+    option :dry_run, aliases: :d, type: :boolean, desc: 'set to false to perform deletion'
+    option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
+    def del_by(pattern)
+      Util.del_by(pattern, dry_run: dry_run, count: count)
+    end
+
+    desc 'drop into a console', 'easy access to helper methods'
+    def console
+      puts "Use `keys '*', count: 1000 to display the first 1000 unique keys matching '*'"
+      puts "Use `del '*', count: 1000 to see how many keys would be deleted for the pattern '*'"
+      puts "Use `del '*', dry_run: false, count: 1000 to delete the first 1000 keys matching '*'"
+      begin
+        require 'pry'
+        Object.include SidekiqUniqueJobs::Util
+        Pry.start
+      rescue LoadError
+        require 'irb'
+        Object.include SidekiqUniqueJobs::Util
+        IRB.start
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/until_timeout.rb

@@ -2,8 +2,11 @@ module SidekiqUniqueJobs
   module Lock
     class UntilTimeout < UntilExecuted
       def unlock(scope)
-        return true if scope.to_sym == :server
-        fail ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
+        if scope.to_sym == :server
+          return true
+        else
+          fail ArgumentError, "#{scope} middleware can't #{__method__} #{unique_key}"
+        end
       end
 
       def execute(_callback)