sidekiq-throttled 0.11.0 → 0.15.0

data/README.md

@@ -1,14 +1,14 @@
 # Sidekiq::Throttled
 
-[![Gem Version](https://badge.fury.io/rb/sidekiq-throttled.svg)](http://rubygems.org/gems/sidekiq-throttled)
-[![Build Status](https://travis-ci.org/sensortower/sidekiq-throttled.svg?branch=master)](https://travis-ci.org/sensortower/sidekiq-throttled)
-[![Code Climate](https://codeclimate.com/github/sensortower/sidekiq-throttled.svg?branch=master)](https://codeclimate.com/github/sensortower/sidekiq-throttled)
-[![Coverage Status](https://coveralls.io/repos/github/sensortower/sidekiq-throttled/badge.svg?branch=master)](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
-[![API Docs](http://inch-ci.org/github/sensortower/sidekiq-throttled.svg?branch=master)](http://inch-ci.org/github/sensortower/sidekiq-throttled)
+[![Latest Version](https://badge.fury.io/rb/sidekiq-throttled.svg)](http://rubygems.org/gems/sidekiq-throttled)
+[![CI Status](https://github.com/sensortower/sidekiq-throttled/workflows/CI/badge.svg?branch=master)](https://github.com/sensortower/sidekiq-throttled/actions?query=workflow%3ACI+branch%3Amaster)
+[![Code Quality](https://codeclimate.com/github/sensortower/sidekiq-throttled.svg?branch=master)](https://codeclimate.com/github/sensortower/sidekiq-throttled)
+[![Code Coverage](https://coveralls.io/repos/github/sensortower/sidekiq-throttled/badge.svg?branch=master)](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
+[![API Docs Quality](http://inch-ci.org/github/sensortower/sidekiq-throttled.svg?branch=master)](http://inch-ci.org/github/sensortower/sidekiq-throttled)
+[![API Docs](https://img.shields.io/badge/yard-docs-blue.svg)](http://www.rubydoc.info/gems/sidekiq-throttled)
 
 Concurrency and threshold throttling for [Sidekiq][sidekiq].
 
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -36,6 +36,9 @@ require "sidekiq/throttled"
 Sidekiq::Throttled.setup!
 ```
 
+Load order can be an issue if you are using other Sidekiq plugins and/or middleware.
+To prevent any problems, add the `.setup!` call to the bottom of your init file.
+
 Once you've done that you can include `Sidekiq::Throttled::Worker` to your
 job classes and configure throttling:
 
@@ -46,12 +49,12 @@ class MyWorker
 
   sidekiq_options :queue => :my_queue
 
-  sidekiq_throttle({
+  sidekiq_throttle(
     # Allow maximum 10 concurrent jobs of this class at a time.
     :concurrency => { :limit => 10 },
     # Allow maximum 1K jobs being processed within one hour window.
     :threshold => { :limit => 1_000, :period => 1.hour }
-  })
+  )
 
   def perform
     # ...
@@ -75,11 +78,11 @@ class MyWorker
 
   sidekiq_options :queue => :my_queue
 
-  sidekiq_throttle({
+  sidekiq_throttle(
     :concurrency => { :limit => 10 },
     :threshold   => { :limit => 100, :period => 1.hour }
     :observer    => MY_OBSERVER
-  })
+  )
 
   def perform(*args)
     # ...
@@ -88,7 +91,7 @@ end
 ```
 
 Observer will receive `strategy, *args` arguments, where `strategy` is a Symbol
-`:concurrency` or `:threshold`, and `*args` are the arguements that were passed
+`:concurrency` or `:threshold`, and `*args` are the arguments that were passed
 to the job.
 
 
@@ -103,10 +106,10 @@ class MyWorker
 
   sidekiq_options :queue => :my_queue
 
-  sidekiq_throttle({
+  sidekiq_throttle(
     # Allow maximum 10 concurrent jobs per user at a time.
     :concurrency => { :limit => 10, :key_suffix => -> (user_id) { user_id } }
-  })
+  )
 
   def perform(user_id)
     # ...
@@ -125,7 +128,7 @@ class MyWorker
 
   sidekiq_options :queue => :my_queue
 
-  sidekiq_throttle({
+  sidekiq_throttle(
     # Allow maximum 1000 concurrent jobs of this class at a time for VIPs and 10 for all other users.
     :concurrency => {
       :limit      => ->(user_id) { User.vip?(user_id) ? 1_000 : 10 },
@@ -136,7 +139,7 @@ class MyWorker
       :limit      => ->(user_id) { User.vip?(user_id) ? 1_000 : 10 },
       :period     => ->(user_id) { User.vip?(user_id) ? 1.hour : 1.day },
       :key_suffix => ->(user_id) { User.vip?(user_id) ? "vip" : "std" }
-  })
+  )
 
   def perform(user_id)
     # ...
@@ -144,6 +147,30 @@ class MyWorker
 end
 ```
 
+You also can use several different keys to throttle one worker.
+
+``` ruby
+class MyWorker
+  include Sidekiq::Worker
+  include Sidekiq::Throttled::Worker
+
+  sidekiq_options :queue => :my_queue
+
+  sidekiq_throttle(
+    # Allow maximum 10 concurrent jobs per project at a time and maximum 2 jobs per user
+    :concurrency => [
+      { :limit => 10, :key_suffix => -> (project_id, user_id) { project_id } },
+      { :limit => 2, :key_suffix => -> (project_id, user_id) { user_id } }
+    ]
+    # For :threshold it works the same
+  )
+
+  def perform(project_id, user_id)
+    # ...
+  end
+end
+```
+
 **NB** Don't forget to specify `:key_suffix` and make it return different values
 if you are using dynamic limit/period options. Otherwise you risk getting into
 some trouble.
@@ -206,9 +233,9 @@ end
 This library aims to support and is [tested against][travis] the following Ruby
 versions:
 
-* Ruby 2.4.x
-* Ruby 2.5.x
 * Ruby 2.6.x
+* Ruby 2.7.x
+* Ruby 3.0.x
 
 If something doesn't work on one of these versions, it's a bug.
 
@@ -231,6 +258,10 @@ This library aims to support work with following [Sidekiq][sidekiq] versions:
 * Sidekiq 5.0.x
 * Sidekiq 5.1.x
 * Sidekiq 5.2.x
+* Sidekiq 6.0.x
+* Sidekiq 6.1.x
+* Sidekiq 6.2.x
+* Sidekiq 6.3.x
 
 
 ## Contributing
@@ -258,7 +289,7 @@ Don't forget to run `appraisal update` after any changes to `Gemfile`.
 
 ## Copyright
 
-Copyright (c) 2015-2018 SensorTower Inc.
+Copyright (c) 2020-2021 Alexey Zapparov, SensorTower Inc.
 See LICENSE.md for further details.
 
 

data/gemfiles/sidekiq_5.0.gemfile

@@ -5,8 +5,9 @@ source "https://rubygems.org"
 gem "appraisal"
 gem "rake"
 gem "rspec"
-gem "rubocop", "~> 0.58.0", require: false
-gem "rubocop-rspec", "~> 1.29.1", require: false
+gem "rubocop", "~> 0.90.0", require: false
+gem "rubocop-performance", "~> 1.8.0", require: false
+gem "rubocop-rspec", "~> 1.43.2", require: false
 gem "sidekiq", "~> 5.0.0"
 
 group :development do
@@ -17,13 +18,13 @@ group :development do
 end
 
 group :test do
+  gem "apparition"
   gem "capybara"
   gem "coveralls", require: false
-  gem "poltergeist"
   gem "puma"
   gem "rack-test"
-  gem "simplecov", ">= 0.9"
-  gem "sinatra", "~> 1.4", ">= 1.4.6"
+  gem "simplecov"
+  gem "sinatra"
   gem "timecop"
 end
 

data/gemfiles/sidekiq_5.1.gemfile

@@ -5,8 +5,9 @@ source "https://rubygems.org"
 gem "appraisal"
 gem "rake"
 gem "rspec"
-gem "rubocop", "~> 0.58.0", require: false
-gem "rubocop-rspec", "~> 1.29.1", require: false
+gem "rubocop", "~> 0.90.0", require: false
+gem "rubocop-performance", "~> 1.8.0", require: false
+gem "rubocop-rspec", "~> 1.43.2", require: false
 gem "sidekiq", "~> 5.1.0"
 
 group :development do
@@ -17,13 +18,13 @@ group :development do
 end
 
 group :test do
+  gem "apparition"
   gem "capybara"
   gem "coveralls", require: false
-  gem "poltergeist"
   gem "puma"
   gem "rack-test"
-  gem "simplecov", ">= 0.9"
-  gem "sinatra", "~> 1.4", ">= 1.4.6"
+  gem "simplecov"
+  gem "sinatra"
   gem "timecop"
 end
 

data/gemfiles/sidekiq_5.2.gemfile

@@ -5,8 +5,9 @@ source "https://rubygems.org"
 gem "appraisal"
 gem "rake"
 gem "rspec"
-gem "rubocop", "~> 0.58.0", require: false
-gem "rubocop-rspec", "~> 1.29.1", require: false
+gem "rubocop", "~> 0.90.0", require: false
+gem "rubocop-performance", "~> 1.8.0", require: false
+gem "rubocop-rspec", "~> 1.43.2", require: false
 gem "sidekiq", "~> 5.2.0"
 
 group :development do
@@ -17,13 +18,13 @@ group :development do
 end
 
 group :test do
+  gem "apparition"
   gem "capybara"
   gem "coveralls", require: false
-  gem "poltergeist"
   gem "puma"
   gem "rack-test"
-  gem "simplecov", ">= 0.9"
-  gem "sinatra", "~> 1.4", ">= 1.4.6"
+  gem "simplecov"
+  gem "sinatra"
   gem "timecop"
 end
 

data/gemfiles/sidekiq_6.0.gemfile

@@ -0,0 +1,31 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "appraisal"
+gem "rake"
+gem "rspec"
+gem "rubocop", "~> 0.90.0", require: false
+gem "rubocop-performance", "~> 1.8.0", require: false
+gem "rubocop-rspec", "~> 1.43.2", require: false
+gem "sidekiq", "~> 6.0.0"
+
+group :development do
+  gem "byebug"
+  gem "guard", require: false
+  gem "guard-rspec", require: false
+  gem "guard-rubocop", require: false
+end
+
+group :test do
+  gem "apparition"
+  gem "capybara"
+  gem "coveralls", require: false
+  gem "puma"
+  gem "rack-test"
+  gem "simplecov"
+  gem "sinatra"
+  gem "timecop"
+end
+
+gemspec path: "../"

data/gemfiles/sidekiq_6.1.gemfile

@@ -0,0 +1,31 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "appraisal"
+gem "rake"
+gem "rspec"
+gem "rubocop", "~> 0.90.0", require: false
+gem "rubocop-performance", "~> 1.8.0", require: false
+gem "rubocop-rspec", "~> 1.43.2", require: false
+gem "sidekiq", "~> 6.1.0"
+
+group :development do
+  gem "byebug"
+  gem "guard", require: false
+  gem "guard-rspec", require: false
+  gem "guard-rubocop", require: false
+end
+
+group :test do
+  gem "apparition"
+  gem "capybara"
+  gem "coveralls", require: false
+  gem "puma"
+  gem "rack-test"
+  gem "simplecov"
+  gem "sinatra"
+  gem "timecop"
+end
+
+gemspec path: "../"

data/gemfiles/sidekiq_6.2.gemfile

@@ -0,0 +1,31 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "appraisal"
+gem "rake"
+gem "rspec"
+gem "rubocop", "~> 0.90.0", require: false
+gem "rubocop-performance", "~> 1.8.0", require: false
+gem "rubocop-rspec", "~> 1.43.2", require: false
+gem "sidekiq", "~> 6.2.0"
+
+group :development do
+  gem "byebug"
+  gem "guard", require: false
+  gem "guard-rspec", require: false
+  gem "guard-rubocop", require: false
+end
+
+group :test do
+  gem "apparition"
+  gem "capybara"
+  gem "coveralls", require: false
+  gem "puma"
+  gem "rack-test"
+  gem "simplecov"
+  gem "sinatra"
+  gem "timecop"
+end
+
+gemspec path: "../"

data/gemfiles/sidekiq_6.3.gemfile

@@ -0,0 +1,31 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "appraisal"
+gem "rake"
+gem "rspec"
+gem "rubocop", "~> 0.90.0", require: false
+gem "rubocop-performance", "~> 1.8.0", require: false
+gem "rubocop-rspec", "~> 1.43.2", require: false
+gem "sidekiq", "~> 6.3.0"
+
+group :development do
+  gem "byebug"
+  gem "guard", require: false
+  gem "guard-rspec", require: false
+  gem "guard-rubocop", require: false
+end
+
+group :test do
+  gem "apparition"
+  gem "capybara"
+  gem "coveralls", require: false
+  gem "puma"
+  gem "rack-test"
+  gem "simplecov"
+  gem "sinatra"
+  gem "timecop"
+end
+
+gemspec path: "../"

data/lib/sidekiq/throttled/communicator/callbacks.rb

@@ -43,6 +43,7 @@ module Sidekiq
         # @return [self]
         def on(event, &handler)
           raise ArgumentError, "No block given" unless handler
+
           @mutex.synchronize { @handlers[event.to_s] << handler }
           self
         end
@@ -54,17 +55,15 @@ module Sidekiq
         # @return [void]
         def run(event, payload = nil)
           @mutex.synchronize do
-            Fiber.new do
+            fiber = Fiber.new do
               @handlers[event.to_s].each do |callback|
-                begin
-                  callback.call(payload)
-                rescue => e
-                  handle_exception(e, {
-                    :context => "sidekiq:throttled"
-                  })
-                end
+                callback.call(payload)
+              rescue => e
+                handle_exception(e, :context => "sidekiq:throttled")
               end
-            end.resume
+            end
+
+            fiber.resume
           end
         end
       end

data/lib/sidekiq/throttled/communicator/listener.rb

@@ -62,7 +62,7 @@ module Sidekiq
         # - `Exception` is recorded to the log and re-raised.
         #
         # @return [void]
-        def listen
+        def listen # rubocop:disable Metrics/MethodLength
           subscribe
         rescue Sidekiq::Shutdown
           @terminated = true
@@ -88,7 +88,7 @@ module Sidekiq
         # @see http://redis.io/commands/subscribe
         # @see Callbacks#run
         # @return [void]
-        def subscribe
+        def subscribe # rubocop:disable Metrics/MethodLength
           Sidekiq.redis do |conn|
             conn.subscribe @channel do |on|
               on.subscribe do
@@ -97,7 +97,7 @@ module Sidekiq
               end
 
               on.message do |_channel, data|
-                message, payload = Marshal.load(data)
+                message, payload = Marshal.load(data) # rubocop:disable Security/MarshalLoad:
                 @callbacks.run("message:#{message}", payload)
               end
             end

data/lib/sidekiq/throttled/expirable_list.rb

@@ -11,7 +11,7 @@ module Sidekiq
     #
     # ## Implementation
     #
-    # Internally list holds an array of arrays. Thus ecah element is a tuple of
+    # Internally list holds an array of arrays. Thus each element is a tuple of
     # monotonic timestamp (when element was added) and element itself:
     #
     #     [

data/lib/sidekiq/throttled/fetch.rb

@@ -12,16 +12,47 @@ module Sidekiq
     #
     # @private
     class Fetch
+      module BulkRequeue
+        # Requeues all given units as a single operation.
+        #
+        # @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
+        # @param [Array<Fetch::UnitOfWork>] units
+        # @return [void]
+        def bulk_requeue(units, _options)
+          return if units.empty?
+
+          Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+          Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
+          Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
+        rescue => e
+          Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
+        end
+      end
+
+      # https://github.com/mperham/sidekiq/commit/fce05c9d4b4c0411c982078a4cf3a63f20f739bc
+      if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.1.0")
+        extend BulkRequeue
+      else
+        include BulkRequeue
+      end
       # Timeout to sleep between fetch retries in case of no job received,
       # as well as timeout to wait for redis to give us something to work.
       TIMEOUT = 2
 
       # Initializes fetcher instance.
+      # @param options [Hash]
+      # @option options [Integer] :throttled_queue_cooldown (TIMEOUT)
+      #   Min delay in seconds before queue will be polled again after
+      #   throttled job.
+      # @option options [Boolean] :strict (false)
+      # @option options [Array<#to_s>] :queue
       def initialize(options)
-        @paused = ExpirableList.new(TIMEOUT)
+        @paused = ExpirableList.new(options.fetch(:throttled_queue_cooldown, TIMEOUT))
 
-        @strict = options[:strict]
-        @queues = options[:queues].map { |q| QueueName.expand q }
+        @strict = options.fetch(:strict, false)
+        @queues = options.fetch(:queues).map { |q| QueueName.expand q }
+
+        raise ArgumentError, "empty :queues" if @queues.empty?
 
         @queues.uniq! if @strict
       end
@@ -42,23 +73,6 @@ module Sidekiq
         nil
       end
 
-      class << self
-        # Requeues all given units as a single operation.
-        #
-        # @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
-        # @param [Array<Fetch::UnitOfWork>] units
-        # @return [void]
-        def bulk_requeue(units, _options)
-          return if units.empty?
-
-          Sidekiq.logger.debug { "Re-queueing terminated jobs" }
-          Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
-          Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
-        rescue => e
-          Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
-        end
-      end
-
       private
 
       # Tries to pop pair of `queue` and job `message` out of sidekiq queues.

data/lib/sidekiq/throttled/queue_name.rb

@@ -7,7 +7,7 @@ module Sidekiq
     # @private
     module QueueName
       # RegExp used to stip out any redisr-namespace prefixes with `queue:`.
-      QUEUE_NAME_PREFIX_RE = /.*queue:/
+      QUEUE_NAME_PREFIX_RE = %r{.*queue:}.freeze
       private_constant :QUEUE_NAME_PREFIX_RE
 
       class << self

data/lib/sidekiq/throttled/registry.rb

@@ -18,20 +18,16 @@ module Sidekiq
 
         # Adds strategy to the registry.
         #
-        # @note prints a warning to STDERR upon duplicate strategy name
         # @param (see Strategy#initialize)
         # @return [Strategy]
         def add(name, **kwargs)
           name = name.to_s
 
-          warn "Duplicate strategy name: #{name}" if @strategies[name]
-
           @strategies[name] = Strategy.new(name, **kwargs)
         end
 
         # Adds alias for existing strategy.
         #
-        # @note prints a warning to STDERR upon duplicate strategy name
         # @param (#to_s) new_name
         # @param (#to_s) old_name
         # @raise [RuntimeError] if no strategy found with `old_name`
@@ -40,7 +36,6 @@ module Sidekiq
           new_name = new_name.to_s
           old_name = old_name.to_s
 
-          warn "Duplicate strategy name: #{new_name}" if @strategies[new_name]
           raise "Strategy not found: #{old_name}" unless @strategies[old_name]
 
           @aliases[new_name] = @strategies[old_name]
@@ -74,6 +69,7 @@ module Sidekiq
         #   @return [Registry]
         def each
           return to_enum(__method__) unless block_given?
+
           @strategies.each { |*args| yield(*args) }
           self
         end
@@ -88,6 +84,7 @@ module Sidekiq
         #   @return [Registry]
         def each_with_static_keys
           return to_enum(__method__) unless block_given?
+
           @strategies.each do |name, strategy|
             yield(name, strategy) unless strategy.dynamic?
           end

data/lib/sidekiq/throttled/strategy/concurrency.rb

@@ -46,12 +46,12 @@ module Sidekiq
           return false unless job_limit
           return true if job_limit <= 0
 
-          kwargs = {
-            :keys => [key(job_args)],
-            :argv => [jid.to_s, job_limit, @ttl, Time.now.to_f]
-          }
+          keys = [key(job_args)]
+          argv = [jid.to_s, job_limit, @ttl, Time.now.to_f]
 
-          Sidekiq.redis { |redis| 1 == SCRIPT.eval(redis, kwargs) }
+          Sidekiq.redis do |redis|
+            1 == SCRIPT.eval(redis, :keys => keys, :argv => argv)
+          end
         end
 
         # @return [Integer] Current count of jobs

data/lib/sidekiq/throttled/strategy/threshold.rb

@@ -48,6 +48,7 @@ module Sidekiq
         # @return [Float] Period in seconds
         def period(job_args = nil)
           return @period.to_f unless @period.respond_to? :call
+
           @period.call(*job_args).to_f
         end
 
@@ -62,12 +63,12 @@ module Sidekiq
           return false unless job_limit
           return true if job_limit <= 0
 
-          kwargs = {
-            :keys => [key(job_args)],
-            :argv => [job_limit, period(job_args), Time.now.to_f]
-          }
+          keys = [key(job_args)]
+          argv = [job_limit, period(job_args), Time.now.to_f]
 
-          Sidekiq.redis { |redis| 1 == SCRIPT.eval(redis, kwargs) }
+          Sidekiq.redis do |redis|
+            1 == SCRIPT.eval(redis, :keys => keys, :argv => argv)
+          end
         end
 
         # @return [Integer] Current count of jobs