sidekiq-robust-job 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 29ab88d61f30457510da2f336be215a6604f2375583b16b7eaf608fe34cb37c5
-  data.tar.gz: c6a174c766334dee3ca28c9dc43b865f1d0f38b8bfce2021e2bcafeaab2b4724
+  metadata.gz: 16456089ded6f826bcb4d0c9e1c2538dbf577a2204b512204f7228553cfe2405
+  data.tar.gz: 96b63fb47b131378d8409c3e0022b78749e8ca9f231ceb469fdd8a21af282416
 SHA512:
-  metadata.gz: 94388670e901388a0958d15715b4371235e6ef81e8dae86dd57f2340734fc911330c656bee9f39b11af897edcce0b6398e1214609fbb7f5de1a09fdcb5498b7d
-  data.tar.gz: 93ec3981822acf52de3b8191d88c9ef4fe904674412a43d498b4d1dd06ad395ef19321e303f72475685ce74696a8e6d18a691c67018e4b7a046163e70ba1bbd1
+  metadata.gz: c78e1907f15a9c881273c4e8f26a0937bd8233267e358aeb374fe91a5b68db28191e9a863a7c7761e5c6ff0994dd6c3ae57cf94c191f93b81b2a1ee88e24715e
+  data.tar.gz: a901ad322c8e4a370ea2bf84e6ceb42c44930c1cd9804c0a23ad2e9230979b8ede1b13466e3dc38561bbe3c79a7d48be57aa40991044bde1f2ca6f8fbff8b0b1

data/Changelog.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## master
+
+## 0.3.0
+- Fix `enqueue_conflict_resolution_strategies` to work well with `until_executing` strategy. Handles corner case when job is being processed, yet we can't enqueue another one
+- Introduce `while_executing` uniqueness strategy, which is basically execution mutex - ensures only that 2 jobs are not executed in parallel, without considering enqueueing uniqueness.
+
 ## 0.2.0
 - Introduce `persist_self_dropped_jobs` config option
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sidekiq-robust-job (0.2.0)
+    sidekiq-robust-job (0.3.0)
       activesupport (>= 5)
       sidekiq (>= 5)
       sidekiq-cron (~> 1)
@@ -9,55 +9,55 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (5.2.4.4)
-      actionview (= 5.2.4.4)
-      activesupport (= 5.2.4.4)
+    actionpack (5.2.4.5)
+      actionview (= 5.2.4.5)
+      activesupport (= 5.2.4.5)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.4.4)
-      activesupport (= 5.2.4.4)
+    actionview (5.2.4.5)
+      activesupport (= 5.2.4.5)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activemodel (5.2.4.4)
-      activesupport (= 5.2.4.4)
-    activerecord (5.2.4.4)
-      activemodel (= 5.2.4.4)
-      activesupport (= 5.2.4.4)
+    activemodel (5.2.4.5)
+      activesupport (= 5.2.4.5)
+    activerecord (5.2.4.5)
+      activemodel (= 5.2.4.5)
+      activesupport (= 5.2.4.5)
       arel (>= 9.0)
-    activesupport (5.2.4.4)
+    activesupport (5.2.4.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     arel (9.0.0)
     builder (3.2.4)
-    concurrent-ruby (1.1.7)
+    concurrent-ruby (1.1.9)
     connection_pool (2.2.3)
     crass (1.0.6)
     diff-lcs (1.4.4)
     erubi (1.10.0)
-    et-orbi (1.2.4)
+    et-orbi (1.2.7)
       tzinfo
     factory_bot (6.1.0)
       activesupport (>= 5.0.0)
     factory_bot_rails (6.1.0)
       factory_bot (~> 6.1.0)
       railties (>= 5.0.0)
-    fugit (1.4.1)
-      et-orbi (~> 1.1, >= 1.1.8)
+    fugit (1.5.3)
+      et-orbi (~> 1, >= 1.2.7)
       raabro (~> 1.4)
-    i18n (1.8.5)
+    i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    loofah (2.7.0)
+    loofah (2.10.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     method_source (1.0.0)
     mini_portile2 (2.4.0)
-    minitest (5.14.2)
+    minitest (5.14.4)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     pg (1.2.3)
@@ -70,9 +70,9 @@ GEM
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (5.2.4.4)
-      actionpack (= 5.2.4.4)
-      activesupport (= 5.2.4.4)
+    railties (5.2.4.5)
+      actionpack (= 5.2.4.5)
+      activesupport (= 5.2.4.5)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.19.0, < 2.0)
@@ -100,13 +100,13 @@ GEM
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
-    sidekiq-cron (1.2.0)
-      fugit (~> 1.1)
+    sidekiq-cron (1.3.0)
+      fugit (>= 1.1)
       sidekiq (>= 4.2.1)
-    thor (1.0.1)
+    thor (1.1.0)
     thread_safe (0.3.6)
     timecop (0.9.2)
-    tzinfo (1.2.8)
+    tzinfo (1.2.9)
       thread_safe (~> 0.1)
 
 PLATFORMS
@@ -124,4 +124,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   2.1.4
+   2.2.22

data/README.md

@@ -145,7 +145,7 @@ Although keep in mind that using this feature comes with some performance penalt
 
 If you have a lot of conflicts within a short period, consider using `perform_in` instead of `perform_async` and add some random number of seconds (ideally, below 1 minute) to make it easier to apply enqueue conflict resolution strategy.
 
-If you enqueue a lot of the same jobs (same class, same arguments) in a short period of time and `drop_self` strategy, you should consider setting `persist_self_dropped_jobs` config option to false. By default, it's true which means that even the jobs that are dropped are persisted, which might be useful for some profiling or even figuring out in the first place that you have an issue like this. However, under such circumstances this is likely to result in heavier queries fetching a lot of rows from the database, causing a high database load.    
+If you enqueue a lot of the same jobs (same class, same arguments) in a short period of time and `drop_self` strategy, you should consider setting `persist_self_dropped_jobs` config option to false. By default, it's true which means that even the jobs that are dropped are persisted, which might be useful for some profiling or even figuring out in the first place that you have an issue like this. However, under such circumstances this is likely to result in heavier queries fetching a lot of rows from the database, causing a high database load.
 
 Here is an example how to use it:
 
@@ -155,8 +155,7 @@ class MyJob
   include Sidekiq::Worker
   include SidekiqRobustJob::SidekiqJobExtensions
 
-  sidekiq_options queue: "critical", enqueue_conflict_resolution_strategy: "drop_self", 
-    persist_self_dropped_jobs: false
+  sidekiq_options queue: "critical", enqueue_conflict_resolution_strategy: "drop_self", persist_self_dropped_jobs: false
 
   def call(user_id)
     User.find(user_id).do_something
@@ -164,6 +163,12 @@ class MyJob
 end
 ```
 
+To make it more likely for self-dropping to kick-in, especially if there is some sort of burst of multiple same jobs being applied at almost the time, consider adding some delay for the execution of the job:
+
+```
+MyJob.perform_in((5..30).to_a.sample.seconds, user_id)
+```
+
 #### Execution Uniqueness (Mutex)
 
 This feature is about handling a "conflict" (determined by a digest generated based on the job class and its arguments) when there is already the "same job" getting executed (i.e. same job class and arguments) at the same time.
@@ -173,6 +178,7 @@ Let's say that there is already a job scheduled to be executed just in a moment
 1. `no_uniqueness` - this is a default when you don't specify anything. In this case, both jobs will be executed.
 2. `until_executed` - One of the jobs acquires mutex using Redlock. When job is finished, it drops other pending jobs (and assigns `dropped_by_job_id` equal to the job that acquired the lock) with the same digest (based on job's class and arguments), and releases the lock. The job that failed to acquire a mutex is rescheduled (not dropped though, just to be on the safe side) and will be executed in the interval determined by `reschedule_interval_in_seconds` (5 seconds by default).
 3. `until_executing` - One of the jobs acquires mutex using Redlock, it drops and assigns `dropped_by_job_id` equal to the job that acquired the lock) other pending jobs with the same digest (based on job's class and arguments) and releases the lock. And then it executes the actual logic behind the job. The job that failed to acquire a lock is rescheduled (not dropped though, just to be on the safe side) and will be executed in the interval determined by `reschedule_interval_in_seconds` (5 seconds by default).
+4. `while_executing` - One of the jobs acquires mutex using Redlock. The other job that failed to acquire a lock is rescheduled (not dropped though, just to be on the safe side) and will be executed in the interval determined by `reschedule_interval_in_seconds` (5 seconds by default). This strategy grants you only execution mutex, unlike `until_executed` or `until_executing` it will not drop any enqueued jobs.
 
 If you want to use this feature, declare in with other Sidekiq options in the job:
 

data/lib/sidekiq/robust/job/version.rb

@@ -1,7 +1,7 @@
 module Sidekiq
   module Robust
     module Job
-      VERSION = "0.2.0"
+      VERSION = "0.3.0"
     end
   end
 end

data/lib/sidekiq_robust_job/enqueue_conflict_resolution_strategy/drop_self.rb

@@ -2,7 +2,7 @@ class SidekiqRobustJob
   class EnqueueConflictResolutionStrategy
     class DropSelf < SidekiqRobustJob::EnqueueConflictResolutionStrategy::Base
       def execute(job)
-        if jobs_repository.unprocessed_for_digest(job.digest, exclude_id: job.id).any?
+        if jobs_repository.not_started_for_digest(job.digest, exclude_id: job.id).any?
           job.drop(dropped_by_job_id: job.id, clock: clock)
         end
       end

data/lib/sidekiq_robust_job/enqueue_conflict_resolution_strategy/replace.rb

@@ -2,7 +2,7 @@ class SidekiqRobustJob
   class EnqueueConflictResolutionStrategy
     class Replace < SidekiqRobustJob::EnqueueConflictResolutionStrategy::Base
       def execute(job)
-        jobs_repository.drop_unprocessed_jobs_by_digest(
+        jobs_repository.drop_not_started_jobs_by_digest(
           dropped_by_job_id: job.id,
           digest: job.digest,
           exclude_id: job.id

data/lib/sidekiq_robust_job/model.rb

@@ -9,6 +9,7 @@ class SidekiqRobustJob
           SidekiqRobustJob::UniquenessStrategy.no_uniqueness,
           SidekiqRobustJob::UniquenessStrategy.until_executing,
           SidekiqRobustJob::UniquenessStrategy.until_executed,
+          SidekiqRobustJob::UniquenessStrategy.while_executing,
         ].map(&:to_s)
       }, on: :create
 

data/lib/sidekiq_robust_job/repository.rb

@@ -34,6 +34,14 @@ class SidekiqRobustJob
         .select { |potentially_missed_job| missed_job_policy.call(potentially_missed_job) }
     end
 
+    def not_started_for_digest(digest, exclude_id:)
+      jobs_database
+        .where(digest: digest)
+        .where(started_at: nil)
+        .where(dropped_at: nil)
+        .where.not(id: exclude_id)
+    end
+
     def unprocessed_for_digest(digest, exclude_id:)
       jobs_database
         .where(digest: digest)
@@ -50,5 +58,14 @@ class SidekiqRobustJob
         end
       end
     end
+
+    def drop_not_started_jobs_by_digest(dropped_by_job_id:, digest:, exclude_id:)
+      transaction do
+        not_started_for_digest(digest, exclude_id: exclude_id).lock!.find_each do |job|
+          job.drop(dropped_by_job_id: dropped_by_job_id)
+          save(job)
+        end
+      end
+    end
   end
 end

data/lib/sidekiq_robust_job/uniqueness_strategy/while_executing.rb

@@ -0,0 +1,15 @@
+class SidekiqRobustJob
+  class UniquenessStrategy
+    class WhileExecuting < SidekiqRobustJob::UniquenessStrategy::Base
+      def execute(job)
+        lock(job) do |locked|
+          if locked
+            perform_job_and_finalize(job)
+          else
+            job.reschedule and return
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_robust_job/uniqueness_strategy.rb

@@ -12,6 +12,10 @@ class SidekiqRobustJob
       :until_executed
     end
 
+    def self.while_executing
+      :while_executing
+    end
+
     attr_reader :locker, :lock_ttl_proc, :jobs_repository, :memory_monitor
     private     :locker, :lock_ttl_proc, :jobs_repository, :memory_monitor
 
@@ -45,6 +49,13 @@ class SidekiqRobustJob
           jobs_repository: jobs_repository,
           memory_monitor: memory_monitor
         )
+      when SidekiqRobustJob::UniquenessStrategy.while_executing
+        SidekiqRobustJob::UniquenessStrategy::WhileExecuting.new(
+          locker: locker,
+          lock_ttl_proc: lock_ttl_proc,
+          jobs_repository: jobs_repository,
+          memory_monitor: memory_monitor
+        )
       else
         raise UnknownStrategyError.new(strategy)
       end

data/lib/sidekiq_robust_job.rb

@@ -20,6 +20,7 @@ require "sidekiq_robust_job/uniqueness_strategy/base"
 require "sidekiq_robust_job/uniqueness_strategy/no_uniqueness"
 require "sidekiq_robust_job/uniqueness_strategy/until_executed"
 require "sidekiq_robust_job/uniqueness_strategy/until_executing"
+require "sidekiq_robust_job/uniqueness_strategy/while_executing"
 require "sidekiq/cron/job"
 require "sidekiq"
 require "active_support/concern"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-robust-job
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-06 00:00:00.000000000 Z
+date: 2022-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -208,6 +208,7 @@ files:
 - lib/sidekiq_robust_job/uniqueness_strategy/no_uniqueness.rb
 - lib/sidekiq_robust_job/uniqueness_strategy/until_executed.rb
 - lib/sidekiq_robust_job/uniqueness_strategy/until_executing.rb
+- lib/sidekiq_robust_job/uniqueness_strategy/while_executing.rb
 - sidekiq-robust-job.gemspec
 homepage: https://github.com/BookingSync/sidekiq-robust-job
 licenses:
@@ -231,7 +232,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: A gem making Sidekiq jobs fully durable with some extra features