sidekiq-throttled-worker 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2003253a226d47aec2d2353889b19e7c5fe7d7819c3e8af9c2b915319edac217
-  data.tar.gz: 7ad3ce4dfb3c855074e0c81ae43a307936d16ddf6ac4a5f75f558acf43b1cb33
+  metadata.gz: 91197aecff6a9c46cb3c1435c85bdf1b74e5be216bbd9a89396fbdb6f00beb8b
+  data.tar.gz: c04884068ab7d1401460dead6edf0c029760dbe89a77d0490ecbbed7164609de
 SHA512:
-  metadata.gz: a82e967d8959c69329f2e5f64cc18808491510e353102aa74bc3246e9a42331384659676f9ce360e076a845ba72b18fd39e8ba2d5c498c86bdb65b8830ae9f63
-  data.tar.gz: 6d8dbe7965c94d647fa23dc4b521e40441e32087f0498966c11bf991b239667bbff00a7fc82bf8f9e87dd36d98081afe3d36b8fb61a05a1763744841fe9fb8a7
+  metadata.gz: 3a7ee4c386dbdae03b4a41aa2a26d70e8825286708e207486d83f9514c91018c8f2ed7fd00ce9cc698c91d0cf4d682a76e3f3296199047c8b045b9bd5d94f3ba
+  data.tar.gz: 7cfa3edc88f5ef7dcfd9932894fbd07b3a541366e4c0b53ca5613035eb5c17c8f2e48ac0cdb76df5a2aa9bd4fd3e85c60ff58057835fc3b14b2d068d7a89f6f4

data/README.md

@@ -2,6 +2,8 @@
 
 Sidekiq concurrency limit per worker in whole cluster,  inspired by [sidekiq-throttled](https://github.com/sensortower/sidekiq-throttled). This gem only support concurrency, and try to avoid use lua in redis to reduce redis CPU usage.
 
+This gem was designed to limit concurrency for some sensitive resources, e.g., databases and http requests, without concurrency limit, sidekiq may crush this services. When some worker exceed concurrency limit, this gem may try to slow down sidekiq process. Concurrency limit should only be use to in extreme case, you should not use this gem to limit business logic
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -29,7 +31,7 @@ Sidekiq::ThrottledWorker.setup!
 
 This gem add two options attr in `sidekiq_options` func：
 
-* `concurrency` : limit concurrency with current worker in whole cluster. Default is  `nil` , which mean on limit 
+* `concurrency` : limit concurrency with current worker in whole cluster. Default is  `nil` , which mean on limit
 * `concurrency_ttl` : max worker run time, and worker may be block completely max for `concurrency_ttl` in  extreme case. Default is 900, in most cases, you do not need change this.
 
 Here is an example worker

data/lib/sidekiq/throttled_worker/fetch.rb

@@ -12,8 +12,12 @@ module Sidekiq
         return work if worker_class.get_sidekiq_options["concurrency"].nil?
 
         if Concurrency.limit?(worker_class, job_hash['jid'])
-          sleep(rand * 0.1) # wait 100ms to reduce redis qps
-          work.requeue
+          job_hash["concurrency_limit_count"] ||= 0
+          job_hash["concurrency_limit_count"] += 1
+          sleep(rand * [(job_hash["concurrency_limit_count"] - 1) * 0.01, 0.1].min)
+          Sidekiq.redis do |conn|
+            conn.lpush("queue:#{work.queue_name}", Sidekiq.dump_json(job_hash)) # put in queue end to auto wait some time in busy sidekiq cluster
+          end
           nil
         else
           work

data/lib/sidekiq/throttled_worker/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module ThrottledWorker
-    VERSION = "0.1.0"
+    VERSION = "0.1.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-throttled-worker
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - xuxiangyang
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-26 00:00:00.000000000 Z
+date: 2020-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq