sidekiq-robust-job 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c4c75298ef6032a3493a8b34871ba84575d1e59fa0d8a21088b3290542b74b3
-  data.tar.gz: 05d1a83d06300c4f273553f07a6ecc38a78f9eddea8539e1d7b85345d53de2f9
+  metadata.gz: 29ab88d61f30457510da2f336be215a6604f2375583b16b7eaf608fe34cb37c5
+  data.tar.gz: c6a174c766334dee3ca28c9dc43b865f1d0f38b8bfce2021e2bcafeaab2b4724
 SHA512:
-  metadata.gz: f9e79a683e532d8078975b9e876db178218a5c5e2937cd734f61c37107f5a1d53b62ee95f8f4d7cf96c7869a970fcf919965420ec75df84029f2d8847574d3ff
-  data.tar.gz: 31067138cd1c737ccc690908b5e75d12fa7d8d5e7f55e086834566ff2e115d4b2664c6b56126880d44468fe6c7953dd3eebd99f49464c8490087b5180104add2
+  metadata.gz: 94388670e901388a0958d15715b4371235e6ef81e8dae86dd57f2340734fc911330c656bee9f39b11af897edcce0b6398e1214609fbb7f5de1a09fdcb5498b7d
+  data.tar.gz: 93ec3981822acf52de3b8191d88c9ef4fe904674412a43d498b4d1dd06ad395ef19321e303f72475685ce74696a8e6d18a691c67018e4b7a046163e70ba1bbd1

data/Changelog.md

@@ -1,4 +1,7 @@
 # Changelog
 
+## 0.2.0
+- Introduce `persist_self_dropped_jobs` config option
+
 ## 0.1.0
 - Initial release

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sidekiq-robust-job (0.1.0)
+    sidekiq-robust-job (0.2.0)
       activesupport (>= 5)
       sidekiq (>= 5)
       sidekiq-cron (~> 1)
@@ -47,7 +47,7 @@ GEM
     factory_bot_rails (6.1.0)
       factory_bot (~> 6.1.0)
       railties (>= 5.0.0)
-    fugit (1.4.0)
+    fugit (1.4.1)
       et-orbi (~> 1.1, >= 1.1.8)
       raabro (~> 1.4)
     i18n (1.8.5)

data/README.md

@@ -57,6 +57,7 @@ class CreateSidekiqJobs < ActiveRecord::Migration[6.0]
       t.datetime "execute_at"
       t.string "sidekiq_jid"
 
+      t.index ["completed_at", "failed_at", "dropped_at"], name: "index_sidekiq_jobs_on_completed_at_and_failed_at_and_dropped_at"
       t.index ["completed_at"], name: "index_sidekiq_jobs_on_completed_at", using: :brin
       t.index ["created_at"], name: "index_sidekiq_jobs_on_created_at", using: :brin
       t.index ["digest"], name: "index_sidekiq_jobs_on_digest"
@@ -144,6 +145,25 @@ Although keep in mind that using this feature comes with some performance penalt
 
 If you have a lot of conflicts within a short period, consider using `perform_in` instead of `perform_async` and add some random number of seconds (ideally, below 1 minute) to make it easier to apply enqueue conflict resolution strategy.
 
+If you enqueue a lot of the same jobs (same class, same arguments) in a short period of time and `drop_self` strategy, you should consider setting `persist_self_dropped_jobs` config option to false. By default, it's true which means that even the jobs that are dropped are persisted, which might be useful for some profiling or even figuring out in the first place that you have an issue like this. However, under such circumstances this is likely to result in heavier queries fetching a lot of rows from the database, causing a high database load.    
+
+Here is an example how to use it:
+
+
+``` rb
+class MyJob
+  include Sidekiq::Worker
+  include SidekiqRobustJob::SidekiqJobExtensions
+
+  sidekiq_options queue: "critical", enqueue_conflict_resolution_strategy: "drop_self", 
+    persist_self_dropped_jobs: false
+
+  def call(user_id)
+    User.find(user_id).do_something
+  end
+end
+```
+
 #### Execution Uniqueness (Mutex)
 
 This feature is about handling a "conflict" (determined by a digest generated based on the job class and its arguments) when there is already the "same job" getting executed (i.e. same job class and arguments) at the same time.
@@ -269,12 +289,19 @@ expect {
 }.to enqueue_sidekiq_robust_job(MyJob).with(user.id).at(5.seconds.from_now)
 ```
 
+
+There is also a negated matcher: `not_enqueue_sidekiq_robust_job`.
+
 ### How to migrate already enqueued jobs when introducing the gem?
 
 This might be a bit tricky. You might consider using new job classes temporarily so that the already existing jobs are performed and the new ones are getting enqueued and then use again the original class with `SidekiqRobustJob::SidekiqJobExtensions` included and `call` method defined.
 
 You can also stop workers, iterate over all existing jobs, re-schedule them (after including the module) and delete them - it's safe because the actual job will still be there, but it will be enqueued this time with job's ID and  `call` method will be used. And you need to delete the original ones as they might have either a different `perform` method signature, or when having the same one, the argument will have a different meaning that job's ID, which can cause an unexpected behavior. This might require a downtime if you are not able to distinguish just based on the arguments of the job between the previous way of executing jobs and the new one. If you are able to, the downtime might not be required, but a lot of jobs can fail due to `perform` method's signature change. However, you can also re-enqueue these jobs and delete them from `RetrySet`.
 
+### Maintenance
+
+It is recommended to periodically remove old jobs for the maximum performance. [Tartarus](https://github.com/BookingSync/tartarus-rb) is a recommended approach for that.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/sidekiq/robust/job/version.rb

@@ -1,7 +1,7 @@
 module Sidekiq
   module Robust
     module Job
-      VERSION = "0.1.0"
+      VERSION = "0.2.0"
     end
   end
 end

data/lib/sidekiq_robust_job/enqueue_conflict_resolution_strategy/drop_self.rb

@@ -4,7 +4,6 @@ class SidekiqRobustJob
       def execute(job)
         if jobs_repository.unprocessed_for_digest(job.digest, exclude_id: job.id).any?
           job.drop(dropped_by_job_id: job.id, clock: clock)
-          jobs_repository.save(job)
         end
       end
     end

data/lib/sidekiq_robust_job/repository.rb

@@ -17,13 +17,17 @@ class SidekiqRobustJob
     end
 
     def save(record)
-      record.save!
+      record.save! if record.changed?
     end
 
     def create(attributes)
       jobs_database.create!(attributes)
     end
 
+    def build(attributes)
+      jobs_database.new(attributes)
+    end
+
     def missed_jobs(missed_job_policy:)
       jobs_database
         .where(completed_at: nil, dropped_at: nil, failed_at: nil)

data/lib/sidekiq_robust_job/sidekiq_job_manager.rb

@@ -53,7 +53,7 @@ class SidekiqRobustJob
     private
 
     def create_job(job_class, *arguments)
-      jobs_repository.create(
+      jobs_repository.build(
         job_class: job_class,
         arguments: Array.wrap(arguments),
         enqueued_at: clock.now,
@@ -64,9 +64,10 @@ class SidekiqRobustJob
         enqueue_conflict_resolution_strategy: job_class.get_sidekiq_options.fetch("enqueue_conflict_resolution_strategy",
           SidekiqRobustJob::EnqueueConflictResolutionStrategy.do_nothing)
       ).tap do |job|
+        jobs_repository.save(job) if persist_job_immediately?(job_class)
         jobs_repository.transaction do
           resolve_potential_conflict_for_enqueueing(job)
-          jobs_repository.save(job)
+          jobs_repository.save(job) if persist_after_resolving_conflict_for_enqueueing(job, job_class)
         end
       end
     end
@@ -76,5 +77,19 @@ class SidekiqRobustJob
         .resolve(job.enqueue_conflict_resolution_strategy)
         .execute(job)
     end
+
+    def persist_job_immediately?(job_class)
+      persist_self_dropped_jobs?(job_class)
+    end
+
+    def persist_after_resolving_conflict_for_enqueueing(job, job_class)
+      return true if persist_self_dropped_jobs?(job_class)
+
+      !job.dropped?
+    end
+
+    def persist_self_dropped_jobs?(job_class)
+      job_class.get_sidekiq_options.fetch("persist_self_dropped_jobs", true)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-robust-job
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-11-18 00:00:00.000000000 Z
+date: 2021-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq