sidekiq-unique-jobs 6.0.0.rc6 → 6.0.0.rc7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a18917b728c376204bb49c0d25e9114c5a512aaff454cb2276fc3de1f9bc9044
-  data.tar.gz: fab057843aa50766cb6adedcdf180aea0e67eba70ed4db8bf700c710621db47a
+  metadata.gz: c90e3f7dec1b958a7516b7fe9aa703622abba4e7713c1e18d3022c6f853822b2
+  data.tar.gz: b3597746d75ce16841f2dc91bd5ba1377850f1341e4bece3478f803f61840979
 SHA512:
-  metadata.gz: c9797381413519ef4901151f3db6087681c292e4d163b23b4f616e4f5da2bcec47356c53a9da9b03333c86eabade31a58786e0c1c14a3aadbffa25841d1f5dcb
-  data.tar.gz: b409a5e4f6f09b912f64f6326da125e9041c2f73964202710e2d396a211acdf5e41e9b72e52aa1b26795bfe98e0309ebbc99d7d6b2d4417e9aaeeadab316e226
+  metadata.gz: fd3fb117d369c9e5ce97d5ea175813e50f76ae59c41cc6aaf91accc9bc613a0cefffd8d20420d854a7378125b0108ddef603011bc0fede46fa0addd7785ef057
+  data.tar.gz: 9d416d37175611a6de8d8dbe9e416218f5439a9760577d7b337df7b0b8f21da2d35c8a0a4ba4846d8490fe26fa467f21f258c5645701cf81fb9e87d276243434

data/.codeclimate.yml

@@ -8,22 +8,21 @@ plugins:
     config:
       languages:
         - ruby
-  # editorconfig:
-  #   enabled: true
-  #   channel: beta
-  #   config:
-  #     editorconfig: .editorconfig
   flog:
     enabled: true
   fixme:
     enabled: true
   markdownlint:
-      enabled: true
+    enabled: true
   reek:
     enabled: true
+    config:
+      file: .reek.yml
   rubocop:
     enabled: true
-    channel: rubocop-0-54
+    channel: rubocop-0-57
+    config:
+      file: .rubocop.yml
 exclude_patterns:
   - Gemfile
   - "*.gemspec"

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,7 +18,7 @@ What happens instead of the expected behavior?
 ```ruby
 class MyWorker
   include Sidekiq::Worker
-  sidekiq_options unique: :until_executed, queue: :undefault
+  sidekiq_options lock: :until_executed, queue: :undefault
   def perform(args); end
 
   def self.unique_args(args)

data/.reek.yml

@@ -10,54 +10,17 @@ detectors:
   DuplicateMethodCall:
     exclude:
     - Sidekiq#self.use_options
+    - SidekiqUniqueJobs::Web#self.registered
   IrresponsibleModule:
+    enabled: false
+  LongParameterList:
+    enabled: true
     exclude:
-    - Sidekiq::Simulator
-    - SidekiqUniqueJobs::Cli
-    - SidekiqUniqueJobs::Client::Middleware
-    - SidekiqUniqueJobs
-    - Array
-    - Hash
-    - SidekiqUniqueJobs::JidMissing
-    - SidekiqUniqueJobs::LockTimeout
-    - SidekiqUniqueJobs::MaxLockTimeMissing
-    - SidekiqUniqueJobs::RunLockFailed
-    - SidekiqUniqueJobs::ScriptError
-    - SidekiqUniqueJobs::UnexpectedValue
-    - SidekiqUniqueJobs::UniqueKeyMissing
-    - SidekiqUniqueJobs::UnknownLock
-    - SidekiqUniqueJobs::Lock::BaseLock
-    - SidekiqUniqueJobs::Lock::UntilAndWhileExecuting
-    - SidekiqUniqueJobs::Lock::UntilExecuted
-    - SidekiqUniqueJobs::Lock::UntilExecuting
-    - SidekiqUniqueJobs::Lock::UntilExpired
-    - SidekiqUniqueJobs::Lock::WhileExecuting
-    - SidekiqUniqueJobs::Lock::WhileExecutingReject
-    - SidekiqUniqueJobs::Lock::WhileExecutingRequeue
-    - SidekiqUniqueJobs::Locksmith
-    - SidekiqUniqueJobs::Middleware
-    - SidekiqUniqueJobs::Normalizer
-    - SidekiqUniqueJobs::Scripts
-    - SidekiqUniqueJobs::Server::Middleware
-    - Sidekiq::Job
-    - Sidekiq::Job::UniqueExtension
-    - Sidekiq::JobSet
-    - Sidekiq::JobSet::UniqueExtension
-    - Sidekiq::Queue
-    - Sidekiq::Queue::UniqueExtension
-    - Sidekiq::ScheduledSet
-    - Sidekiq::ScheduledSet::UniqueExtension
-    - Sidekiq::SortedEntry
-    - Sidekiq::SortedEntry::UniqueExtension
-    - Sidekiq
-    - Sidekiq::Worker
-    - Sidekiq::Worker::ClassMethods
-    - Sidekiq::Worker::Overrides
-    - Sidekiq::Worker::Overrides::Testing
-    - SidekiqUniqueJobs::Timeout::Calculator
-    - SidekiqUniqueJobs::Timeout
-    - SidekiqUniqueJobs::Unlockable
-    - SidekiqUniqueJobs::Util
+    - initialize
+    - Hash#slice
+    - SidekiqUniqueJobs::Locksmith#create_lock
+    - SidekiqUniqueJobs::Locksmith#expire_when_necessary
+    - SidekiqUniqueJobs::Client::Middleware#call
   TooManyStatements:
     exclude:
     - initialize
@@ -68,6 +31,9 @@ detectors:
     - SidekiqUniqueJobs::Server::Middleware#call
     - SidekiqUniqueJobs::UniqueArgs#filtered_args
     - SidekiqUniqueJobs::Util#del
+    - SidekiqUniqueJobs::Digests#batch_delete
+    - SidekiqUniqueJobs::Digests#delete_by_pattern
+    - SidekiqUniqueJobs::Web#self.registered
   UncommunicativeVariableName:
     exclude:
     - Hash#slice
@@ -84,8 +50,9 @@ detectors:
     exclude:
     - Hash#slice
     - Hash#slice!
-    - SidekiqUniqueJobs::Lock::WhileExecutingReject#deadset_kill?
+    - SidekiqUniqueJobs::OnConflict::Reject#deadset_kill?
     - SidekiqUniqueJobs::SidekiqWorkerMethods#worker_method_defined?
+    - SidekiqUniqueJobs::Web::Helpers#redirect_to
   MissingSafeMethod:
     exclude:
     - Array
@@ -93,15 +60,17 @@ detectors:
     enabled: false
   FeatureEnvy:
     exclude:
-    - SidekiqUniqueJobs::Lock::WhileExecutingReject#push_to_deadset
+    - SidekiqUniqueJobs::OnConflict::Reject#push_to_deadset
     - SidekiqUniqueJobs::Logging#debug_item
     - SidekiqUniqueJobs::Util#batch_delete
+    - SidekiqUniqueJobs::Digests#batch_delete
   NestedIterators:
     exclude:
     - SidekiqUniqueJobs::Locksmith#create_lock
     - SidekiqUniqueJobs::Middleware#configure_client_middleware
     - SidekiqUniqueJobs::Middleware#configure_server_middleware
     - SidekiqUniqueJobs::Util#batch_delete
+    - SidekiqUniqueJobs::Digests#batch_delete
   TooManyInstanceVariables:
     exclude:
     - SidekiqUniqueJobs::Locksmith

data/.rubocop.yml

@@ -64,6 +64,9 @@ RSpec/ExampleLength:
 RSpec/ExpectActual:
   Enabled: false
 
+RSpec/ExpectChange:
+  Enabled: false
+
 RSpec/ExpectInHook:
   Enabled: false
 

data/.yardopts

@@ -0,0 +1,7 @@
+  --no-private
+  --exclude lib/sidekiq_unique_jobs/testing.rb
+  --exclude lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb
+  --exclude lib/sidekiq_unique_jobs/middleware.rb
+  --exclude lib/sidekiq_unique_jobs/core_ext.rb
+  --exclude lib/sidekiq_unique_jobs/cli.rb
+

data/CHANGELOG.md

@@ -11,6 +11,8 @@
 - Removed concurrency for now (a0cff5bc42edbe7190d6ede7e7f845074d2d7af6)
 - Improved integration testing for locks
 - Totally delete the hash that was growing out of proportion
+- Adds a sidekiq web extension for viewing and deleting unique digests
+- Renamed the configuration `unique:` to `lock:` (still backwards compatible)
 
 ## v5.1.0
 

data/README.md

@@ -6,6 +6,7 @@
 * [Documentation](#documentation)
 * [Requirements](#requirements)
 * [Installation](#installation)
+* [Support Me](#support-me)
 * [General Information](#general-information)
 * [Options](#options)
    * [Lock Expiration](#lock-expiration)
@@ -18,15 +19,13 @@
    * [Until Timeout](#until-timeout)
    * [Unique Until And While Executing](#unique-until-and-while-executing)
    * [While Executing](#while-executing)
+* [Conflict Strategy](#conflict-strategy)
 * [Usage](#usage)
    * [Finer Control over Uniqueness](#finer-control-over-uniqueness)
    * [After Unlock Callback](#after-unlock-callback)
    * [Logging](#logging)
 * [Debugging](#debugging)
-   * [Console](#console)
-      * [List Unique Keys](#list-unique-keys)
-      * [Remove Unique Keys](#remove-unique-keys)
-   * [Command Line](#command-line)
+   * [Sidekiq Web](#sidekiq-web)
 * [Communication](#communication)
 * [Testing](#testing)
 * [Contributing](#contributing)
@@ -65,6 +64,11 @@ Or install it yourself as:
 
     $ gem install sidekiq-unique-jobs
 
+
+## Support Me
+
+Want to show me some ❤️ for the hard work I do on this gem? You can use the following PayPal link https://paypal.me/mhenrixon. Any amount is welcome and let me tell you it feels good to be appreciated. Even a dollar makes me super excited about all of this.
+
 ## General Information
 
 See [Interaction w/ Sidekiq](https://github.com/mhenrixon/sidekiq-unique-jobs/wiki/How-this-gem-interacts-with-Sidekiq) on how the gem interacts with Sidekiq.
@@ -150,7 +154,7 @@ Locks from when the client pushes the job to the queue. Will be unlocked before
 **NOTE** this is probably not so good for jobs that shouldn't be running simultaneously (aka slow jobs).
 
 ```ruby
-sidekiq_options unique: :until_executing
+sidekiq_options lock: :until_executing
 ```
 
 ### Until Executed
@@ -158,7 +162,7 @@ sidekiq_options unique: :until_executing
 Locks from when the client pushes the job to the queue. Will be unlocked when the server has successfully processed the job.
 
 ```ruby
-sidekiq_options unique: :until_executed
+sidekiq_options lock: :until_executed
 ```
 
 ### Until Timeout
@@ -166,7 +170,7 @@ sidekiq_options unique: :until_executed
 Locks from when the client pushes the job to the queue. Will be unlocked when the specified timeout has been reached.
 
 ```ruby
-sidekiq_options unique: :until_expired
+sidekiq_options lock: :until_expired
 ```
 
 ### Unique Until And While Executing
@@ -174,7 +178,7 @@ sidekiq_options unique: :until_expired
 Locks when the client pushes the job to the queue. The queue will be unlocked when the server starts processing the job. The server then goes on to creating a runtime lock for the job to prevent simultaneous jobs from being executed. As soon as the server starts processing a job, the client can push the same job to the queue.
 
 ```ruby
-sidekiq_options unique: :until_and_while_executing
+sidekiq_options lock: :until_and_while_executing
 ```
 
 ### While Executing
@@ -184,7 +188,7 @@ With this lock type it is possible to put any number of these jobs on the queue,
 **NOTE** Unless this job is configured with a `lock_timeout: nil` or `lock_timeout: > 0` then all jobs that are attempted to be executed will just be dropped without waiting.
 
 ```ruby
-sidekiq_options unique: :while_executing, lock_timeout: nil
+sidekiq_options lock: :while_executing, lock_timeout: nil
 ```
 
 There is an example of this to try it out in the `rails_example` application. Run `foreman start` in the root of the directory and open the url: `localhost:5000/work/duplicate_while_executing`.
@@ -206,12 +210,42 @@ In the console you should see something like:
 10:33:04 worker.1 | 2017-04-23T08:33:04.973Z 84404 TID-ougq8cs8s WhileExecutingWorker JID-9e197460c067b22eb1b5d07f INFO: done: 40.014 sec
 ```
 
+## Conflict Strategy
+
+Decides how we handle conflict. We can either reject the job to the dead queue or reschedule it. Both are useful for jobs that absolutely need to run and have been configured to use the lock `WhileExecuting` that is used only by the sidekiq server process. 
+
+The last one is log which can be be used with the lock `UntilExecuted` and `UntilExpired`. Now we write a log entry saying the job could not be pushed because it is a duplicate of another job with the same arguments
+
+### Raise
+
+This strategy is intended to be used with `WhileExecuting`. Basically it will allow us to let the server process crash with a specific error message and be retried without messing up the Sidekiq stats.
+
+`sidekiq_options lock: :while_executing, on_conflict: :raise, retry: 10`
+
+### Reject
+
+This strategy is intended to be used with `WhileExecuting` and will push the job to the dead queue on conflict.
+
+`sidekiq_options lock: :while_executing, on_conflict: :reject`
+
+### Reschedule
+
+This strategy is intended to be used with `WhileExecuting` and will delay the job to be tried again in 5 seconds. This will mess up the sidekiq stats but will prevent exceptions from being logged and confuse your sysadmins.
+
+`sidekiq_options lock: :while_executing, on_conflict: :reschedule`
+
+### Log
+
+This strategy is intended to be used with `UntilExecuted` and `UntilExpired`. It will log a line about that this is job is a duplicate of another.
+
+`sidekiq_options lock: :until_executed, on_conflict: :log`
+
 ## Usage
 
 All that is required is that you specifically set the sidekiq option for _unique_ to a valid value like below:
 
 ```ruby
-sidekiq_options unique: :while_executing
+sidekiq_options lock: :while_executing
 ```
 
 Requiring the gem in your gemfile should be sufficient to enable unique jobs.
@@ -227,7 +261,7 @@ The method or the proc can return a modified version of args without the transie
 ```ruby
 class UniqueJobWithFilterMethod
   include Sidekiq::Worker
-  sidekiq_options unique: :until_and_while_executing,
+  sidekiq_options lock: :until_and_while_executing,
                   unique_args: :unique_args # this is default and will be used if such a method is defined
 
   def self.unique_args(args)
@@ -240,7 +274,7 @@ end
 
 class UniqueJobWithFilterProc
   include Sidekiq::Worker
-  sidekiq_options unique: :until_executed,
+  sidekiq_options lock: :until_executed,
                   unique_args: ->(args) { [ args.first ] }
 
   ...
@@ -253,7 +287,7 @@ It is also quite possible to ensure different types of unique args based on cont
 ```ruby
 class UniqueJobWithFilterMethod
   include Sidekiq::Worker
-  sidekiq_options unique: :until_and_while_executing, unique_args: :unique_args
+  sidekiq_options lock: :until_and_while_executing, unique_args: :unique_args
 
   def self.unique_args(args)
     if Sidekiq::ProcessSet.new.size > 1
@@ -277,7 +311,7 @@ If you need to perform any additional work after the lock has been released you
 ```ruby
 class UniqueJobWithFilterMethod
   include Sidekiq::Worker
-  sidekiq_options unique: :while_executing,
+  sidekiq_options lock: :while_executing,
 
   def after_unlock
    # block has yielded and lock is released
@@ -293,7 +327,7 @@ To see logging in sidekiq when duplicate payload has been filtered out you can e
 ```ruby
 class UniqueJobWithFilterMethod
   include Sidekiq::Worker
-  sidekiq_options unique: :while_executing,
+  sidekiq_options lock: :while_executing,
                   log_duplicate_payload: true
 
   ...
@@ -305,21 +339,29 @@ end
 
 There are two ways to display and remove keys regarding uniqueness. The console way and the command line way.
 
-### Console
+### Sidekiq Web
+
+To use the web extension you need to require it in your routes.
+
+```ruby
+# app/config/routes.rb
+require 'sidekiq-unique-jobs/web'
+mount Sidekiq::Web, at: '/sidekiq'
+```
 
-Start the console with the following command `bundle exec uniquejobs console`.
+There is no need to `require 'sidekiq/web'` since `sidekiq_unique_jobs/web` 
+already does this.
 
-#### List Unique Keys
+To filter/search for keys we can use the wildcard `*`. If we have a unique digest `'uniquejobs:9e9b5ce5d423d3ea470977004b50ff84` we can search for it by enter `*ff84` and it should return all digests that end with `ff84`.
 
-`keys '*', 100`
 
-#### Remove Unique Keys
+#### Show Unique Digests
 
-`del '*', 100, false` the dry_run and count parameters are both required. This is to have some type of protection against clearing out all uniqueness.
+![Unique Digests](assets/unique_digests_1.png)
 
-### Command Line
+#### Show keys for digest
 
-`bundle exec uniquejobs` displays help on how to use the unique jobs command line.
+![Unique Digests](assets/unique_digests_2.png)
 
 ## Communication
 

data/examples/another_unique_job.rb

@@ -4,8 +4,10 @@
 
 class AnotherUniqueJob
   include Sidekiq::Worker
-  sidekiq_options queue: :working2, retry: 1, backtrace: 10,
-                  unique: :until_executed
+  sidekiq_options backtrace: 10,
+                  lock: :until_executed,
+                  queue: :working2,
+                  retry: 1
 
   def perform(args)
     args

data/examples/custom_queue_job_with_filter_method.rb

@@ -5,7 +5,7 @@
 require_relative 'custom_queue_job'
 
 class CustomQueueJobWithFilterMethod < CustomQueueJob
-  sidekiq_options unique: :until_executed, unique_args: :args_filter
+  sidekiq_options lock: :until_executed, unique_args: :args_filter
 
   def self.args_filter(args)
     args.first

data/examples/custom_queue_job_with_filter_proc.rb

@@ -7,7 +7,7 @@ require_relative './custom_queue_job'
 class CustomQueueJobWithFilterProc < CustomQueueJob
   # slightly contrived example of munging args to the
   # worker and removing a random bit.
-  sidekiq_options unique: :until_expired,
+  sidekiq_options lock: :until_expired,
                   unique_args: (lambda do |args|
                     options = args.extract_options!
                     options.delete('random')

data/examples/expiring_job.rb

@@ -4,7 +4,7 @@
 
 class ExpiringJob
   include Sidekiq::Worker
-  sidekiq_options unique: :until_executed, lock_expiration: 10 * 60
+  sidekiq_options lock: :until_executed, lock_expiration: 10 * 60
 
   def perform(one, two)
     [one, two]

data/examples/inline_worker.rb

@@ -4,7 +4,7 @@
 
 class InlineWorker
   include Sidekiq::Worker
-  sidekiq_options unique: :while_executing, lock_timeout: 5
+  sidekiq_options lock: :while_executing, lock_timeout: 5
 
   def perform(one)
     TestClass.run(one)