sidekiq-throttled 0.10.0 → 0.14.1

data/gemfiles/sidekiq_5.0.gemfile

@@ -5,8 +5,9 @@ source "https://rubygems.org"
 gem "appraisal"
 gem "rake"
 gem "rspec"
-gem "rubocop", "~> 0.58.0", require: false
-gem "rubocop-rspec", "~> 1.29.1", require: false
+gem "rubocop", "~> 0.82.0", require: false
+gem "rubocop-performance", "~>1.5.2", require: false
+gem "rubocop-rspec", "~> 1.39.0", require: false
 gem "sidekiq", "~> 5.0.0"
 
 group :development do
@@ -17,13 +18,13 @@ group :development do
 end
 
 group :test do
+  gem "apparition"
   gem "capybara"
   gem "coveralls", require: false
-  gem "poltergeist"
   gem "puma"
   gem "rack-test"
-  gem "simplecov", ">= 0.9"
-  gem "sinatra", "~> 1.4", ">= 1.4.6"
+  gem "simplecov"
+  gem "sinatra"
   gem "timecop"
 end
 

data/gemfiles/sidekiq_5.1.gemfile

@@ -5,8 +5,9 @@ source "https://rubygems.org"
 gem "appraisal"
 gem "rake"
 gem "rspec"
-gem "rubocop", "~> 0.58.0", require: false
-gem "rubocop-rspec", "~> 1.29.1", require: false
+gem "rubocop", "~> 0.82.0", require: false
+gem "rubocop-performance", "~>1.5.2", require: false
+gem "rubocop-rspec", "~> 1.39.0", require: false
 gem "sidekiq", "~> 5.1.0"
 
 group :development do
@@ -17,13 +18,13 @@ group :development do
 end
 
 group :test do
+  gem "apparition"
   gem "capybara"
   gem "coveralls", require: false
-  gem "poltergeist"
   gem "puma"
   gem "rack-test"
-  gem "simplecov", ">= 0.9"
-  gem "sinatra", "~> 1.4", ">= 1.4.6"
+  gem "simplecov"
+  gem "sinatra"
   gem "timecop"
 end
 

data/gemfiles/sidekiq_5.2.gemfile

@@ -5,8 +5,9 @@ source "https://rubygems.org"
 gem "appraisal"
 gem "rake"
 gem "rspec"
-gem "rubocop", "~> 0.58.0", require: false
-gem "rubocop-rspec", "~> 1.29.1", require: false
+gem "rubocop", "~> 0.82.0", require: false
+gem "rubocop-performance", "~>1.5.2", require: false
+gem "rubocop-rspec", "~> 1.39.0", require: false
 gem "sidekiq", "~> 5.2.0"
 
 group :development do
@@ -17,13 +18,13 @@ group :development do
 end
 
 group :test do
+  gem "apparition"
   gem "capybara"
   gem "coveralls", require: false
-  gem "poltergeist"
   gem "puma"
   gem "rack-test"
-  gem "simplecov", ">= 0.9"
-  gem "sinatra", "~> 1.4", ">= 1.4.6"
+  gem "simplecov"
+  gem "sinatra"
   gem "timecop"
 end
 

data/gemfiles/sidekiq_6.0.gemfile

@@ -0,0 +1,31 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "appraisal"
+gem "rake"
+gem "rspec"
+gem "rubocop", "~> 0.82.0", require: false
+gem "rubocop-performance", "~>1.5.2", require: false
+gem "rubocop-rspec", "~> 1.39.0", require: false
+gem "sidekiq", "~> 6.0.0"
+
+group :development do
+  gem "byebug"
+  gem "guard", require: false
+  gem "guard-rspec", require: false
+  gem "guard-rubocop", require: false
+end
+
+group :test do
+  gem "apparition"
+  gem "capybara"
+  gem "coveralls", require: false
+  gem "puma"
+  gem "rack-test"
+  gem "simplecov"
+  gem "sinatra"
+  gem "timecop"
+end
+
+gemspec path: "../"

data/gemfiles/sidekiq_6.1.gemfile

@@ -0,0 +1,31 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "appraisal"
+gem "rake"
+gem "rspec"
+gem "rubocop", "~> 0.82.0", require: false
+gem "rubocop-performance", "~>1.5.2", require: false
+gem "rubocop-rspec", "~> 1.39.0", require: false
+gem "sidekiq", "~> 6.1.0"
+
+group :development do
+  gem "byebug"
+  gem "guard", require: false
+  gem "guard-rspec", require: false
+  gem "guard-rubocop", require: false
+end
+
+group :test do
+  gem "apparition"
+  gem "capybara"
+  gem "coveralls", require: false
+  gem "puma"
+  gem "rack-test"
+  gem "simplecov"
+  gem "sinatra"
+  gem "timecop"
+end
+
+gemspec path: "../"

data/lib/sidekiq/throttled/communicator/callbacks.rb

@@ -43,6 +43,7 @@ module Sidekiq
         # @return [self]
         def on(event, &handler)
           raise ArgumentError, "No block given" unless handler
+
           @mutex.synchronize { @handlers[event.to_s] << handler }
           self
         end
@@ -52,19 +53,19 @@ module Sidekiq
         # @param [#to_s] event
         # @param [Object] payload
         # @return [void]
-        def run(event, payload = nil)
+        def run(event, payload = nil) # rubocop:disable Metrics/MethodLength
           @mutex.synchronize do
-            Fiber.new do
+            fiber = Fiber.new do
               @handlers[event.to_s].each do |callback|
                 begin
                   callback.call(payload)
                 rescue => e
-                  handle_exception(e, {
-                    :context => "sidekiq:throttled"
-                  })
+                  handle_exception(e, :context => "sidekiq:throttled")
                 end
               end
-            end.resume
+            end
+
+            fiber.resume
           end
         end
       end

data/lib/sidekiq/throttled/communicator/listener.rb

@@ -62,7 +62,7 @@ module Sidekiq
         # - `Exception` is recorded to the log and re-raised.
         #
         # @return [void]
-        def listen
+        def listen # rubocop:disable Metrics/MethodLength
           subscribe
         rescue Sidekiq::Shutdown
           @terminated = true
@@ -88,7 +88,7 @@ module Sidekiq
         # @see http://redis.io/commands/subscribe
         # @see Callbacks#run
         # @return [void]
-        def subscribe
+        def subscribe # rubocop:disable Metrics/MethodLength
           Sidekiq.redis do |conn|
             conn.subscribe @channel do |on|
               on.subscribe do
@@ -97,7 +97,7 @@ module Sidekiq
               end
 
               on.message do |_channel, data|
-                message, payload = Marshal.load(data)
+                message, payload = Marshal.load(data) # rubocop:disable Security/MarshalLoad:
                 @callbacks.run("message:#{message}", payload)
               end
             end

data/lib/sidekiq/throttled/expirable_list.rb

@@ -11,7 +11,7 @@ module Sidekiq
     #
     # ## Implementation
     #
-    # Internally list holds an array of arrays. Thus ecah element is a tuple of
+    # Internally list holds an array of arrays. Thus each element is a tuple of
     # monotonic timestamp (when element was added) and element itself:
     #
     #     [

data/lib/sidekiq/throttled/fetch.rb

@@ -12,16 +12,47 @@ module Sidekiq
     #
     # @private
     class Fetch
+      module BulkRequeue
+        # Requeues all given units as a single operation.
+        #
+        # @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
+        # @param [Array<Fetch::UnitOfWork>] units
+        # @return [void]
+        def bulk_requeue(units, _options)
+          return if units.empty?
+
+          Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+          Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
+          Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
+        rescue => e
+          Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
+        end
+      end
+
+      # https://github.com/mperham/sidekiq/commit/fce05c9d4b4c0411c982078a4cf3a63f20f739bc
+      if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.1.0")
+        extend BulkRequeue
+      else
+        include BulkRequeue
+      end
       # Timeout to sleep between fetch retries in case of no job received,
       # as well as timeout to wait for redis to give us something to work.
       TIMEOUT = 2
 
       # Initializes fetcher instance.
+      # @param options [Hash]
+      # @option options [Integer] :throttled_queue_cooldown (TIMEOUT)
+      #   Min delay in seconds before queue will be polled again after
+      #   throttled job.
+      # @option options [Boolean] :strict (false)
+      # @option options [Array<#to_s>] :queue
       def initialize(options)
-        @paused = ExpirableList.new(TIMEOUT)
+        @paused = ExpirableList.new(options.fetch(:throttled_queue_cooldown, TIMEOUT))
 
-        @strict = options[:strict]
-        @queues = options[:queues].map { |q| QueueName.expand q }
+        @strict = options.fetch(:strict, false)
+        @queues = options.fetch(:queues).map { |q| QueueName.expand q }
+
+        raise ArgumentError, "empty :queues" if @queues.empty?
 
         @queues.uniq! if @strict
       end
@@ -42,23 +73,6 @@ module Sidekiq
         nil
       end
 
-      class << self
-        # Requeues all given units as a single operation.
-        #
-        # @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
-        # @param [Array<Fetch::UnitOfWork>] units
-        # @return [void]
-        def bulk_requeue(units, _options)
-          return if units.empty?
-
-          Sidekiq.logger.debug { "Re-queueing terminated jobs" }
-          Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
-          Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
-        rescue => e
-          Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
-        end
-      end
-
       private
 
       # Tries to pop pair of `queue` and job `message` out of sidekiq queues.

data/lib/sidekiq/throttled/queue_name.rb

@@ -7,7 +7,7 @@ module Sidekiq
     # @private
     module QueueName
       # RegExp used to stip out any redisr-namespace prefixes with `queue:`.
-      QUEUE_NAME_PREFIX_RE = /.*queue:/
+      QUEUE_NAME_PREFIX_RE = %r{.*queue:}.freeze
       private_constant :QUEUE_NAME_PREFIX_RE
 
       class << self

data/lib/sidekiq/throttled/registry.rb

@@ -18,20 +18,16 @@ module Sidekiq
 
         # Adds strategy to the registry.
         #
-        # @note prints a warning to STDERR upon duplicate strategy name
         # @param (see Strategy#initialize)
         # @return [Strategy]
         def add(name, **kwargs)
           name = name.to_s
 
-          warn "Duplicate strategy name: #{name}" if @strategies[name]
-
           @strategies[name] = Strategy.new(name, **kwargs)
         end
 
         # Adds alias for existing strategy.
         #
-        # @note prints a warning to STDERR upon duplicate strategy name
         # @param (#to_s) new_name
         # @param (#to_s) old_name
         # @raise [RuntimeError] if no strategy found with `old_name`
@@ -40,7 +36,6 @@ module Sidekiq
           new_name = new_name.to_s
           old_name = old_name.to_s
 
-          warn "Duplicate strategy name: #{new_name}" if @strategies[new_name]
           raise "Strategy not found: #{old_name}" unless @strategies[old_name]
 
           @aliases[new_name] = @strategies[old_name]
@@ -74,6 +69,7 @@ module Sidekiq
         #   @return [Registry]
         def each
           return to_enum(__method__) unless block_given?
+
           @strategies.each { |*args| yield(*args) }
           self
         end
@@ -88,6 +84,7 @@ module Sidekiq
         #   @return [Registry]
         def each_with_static_keys
           return to_enum(__method__) unless block_given?
+
           @strategies.each do |name, strategy|
             yield(name, strategy) unless strategy.dynamic?
           end

data/lib/sidekiq/throttled/strategy/concurrency.rb

@@ -46,12 +46,12 @@ module Sidekiq
           return false unless job_limit
           return true if job_limit <= 0
 
-          kwargs = {
-            :keys => [key(job_args)],
-            :argv => [jid.to_s, job_limit, @ttl, Time.now.to_f]
-          }
+          keys = [key(job_args)]
+          argv = [jid.to_s, job_limit, @ttl, Time.now.to_f]
 
-          Sidekiq.redis { |redis| 1 == SCRIPT.eval(redis, kwargs) }
+          Sidekiq.redis do |redis|
+            1 == SCRIPT.eval(redis, :keys => keys, :argv => argv)
+          end
         end
 
         # @return [Integer] Current count of jobs

data/lib/sidekiq/throttled/strategy/threshold.rb

@@ -48,6 +48,7 @@ module Sidekiq
         # @return [Float] Period in seconds
         def period(job_args = nil)
           return @period.to_f unless @period.respond_to? :call
+
           @period.call(*job_args).to_f
         end
 
@@ -62,12 +63,12 @@ module Sidekiq
           return false unless job_limit
           return true if job_limit <= 0
 
-          kwargs = {
-            :keys => [key(job_args)],
-            :argv => [job_limit, period(job_args), Time.now.to_f]
-          }
+          keys = [key(job_args)]
+          argv = [job_limit, period(job_args), Time.now.to_f]
 
-          Sidekiq.redis { |redis| 1 == SCRIPT.eval(redis, kwargs) }
+          Sidekiq.redis do |redis|
+            1 == SCRIPT.eval(redis, :keys => keys, :argv => argv)
+          end
         end
 
         # @return [Integer] Current count of jobs

data/lib/sidekiq/throttled/strategy.rb

@@ -2,6 +2,7 @@
 
 # internal
 require "sidekiq/throttled/errors"
+require "sidekiq/throttled/strategy_collection"
 require "sidekiq/throttled/strategy/concurrency"
 require "sidekiq/throttled/strategy/threshold"
 
@@ -19,28 +20,31 @@ module Sidekiq
       #   @return [Strategy::Threshold, nil]
       attr_reader :threshold
 
+      # @!attribute [r] observer
+      #   @return [Proc, nil]
+      attr_reader :observer
+
       # @param [#to_s] name
       # @param [Hash] concurrency Concurrency options.
       #   See keyword args of {Strategy::Concurrency#initialize} for details.
       # @param [Hash] threshold Threshold options.
       #   See keyword args of {Strategy::Threshold#initialize} for details.
       # @param [#call] key_suffix Dynamic key suffix generator.
-      def initialize(name, concurrency: nil, threshold: nil, key_suffix: nil)
-        key = "throttled:#{name}"
+      # @param [#call] observer Process called after throttled.
+      def initialize(name, concurrency: nil, threshold: nil, key_suffix: nil, observer: nil) # rubocop:disable Metrics/MethodLength
+        @observer = observer
 
-        @concurrency =
-          if concurrency
-            concurrency[:key_suffix] ||= key_suffix
-            Concurrency.new(key, **concurrency)
-          end
+        @concurrency = StrategyCollection.new(concurrency,
+          :strategy   => Concurrency,
+          :name       => name,
+          :key_suffix => key_suffix)
 
-        @threshold =
-          if threshold
-            threshold[:key_suffix] ||= key_suffix
-            Threshold.new(key, **threshold)
-          end
+        @threshold = StrategyCollection.new(threshold,
+          :strategy   => Threshold,
+          :name       => name,
+          :key_suffix => key_suffix)
 
-        return if @concurrency || @threshold
+        return if @concurrency.any? || @threshold.any?
 
         raise ArgumentError, "Neither :concurrency nor :threshold given"
       end
@@ -55,9 +59,14 @@ module Sidekiq
 
       # @return [Boolean] whenever job is throttled or not.
       def throttled?(jid, *job_args)
-        return true if @concurrency&.throttled?(jid, *job_args)
+        if @concurrency&.throttled?(jid, *job_args)
+          @observer&.call(:concurrency, *job_args)
+          return true
+        end
 
         if @threshold&.throttled?(*job_args)
+          @observer&.call(:threshold, *job_args)
+
           finalize!(jid, *job_args)
           return true
         end

data/lib/sidekiq/throttled/strategy_collection.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# internal
+module Sidekiq
+  module Throttled
+    # Collection which transparently group several meta-strategies of one kind
+    #
+    # @private
+    class StrategyCollection
+      include Enumerable
+
+      attr_reader :strategies
+
+      # @param [Hash, Array, nil] strategies Concurrency or Threshold options
+      #   or array of options.
+      #   See keyword args of {Strategy::Concurrency#initialize} for details.
+      #   See keyword args of {Strategy::Threshold#initialize} for details.
+      # @param [Class] strategy class of strategy: Concurrency or Threshold
+      # @param [#to_s] name
+      # @param [#call] key_suffix Dynamic key suffix generator.
+      def initialize(strategies, strategy:, name:, key_suffix:)
+        strategies = (strategies.is_a?(Hash) ? [strategies] : Array(strategies))
+        @strategies = strategies.map do |options|
+          make_strategy(strategy, name, key_suffix, options)
+        end
+      end
+
+      # @param [#call] block
+      # Iterates each strategy in collection
+      def each(&block)
+        @strategies.each(&block)
+      end
+
+      # @return [Boolean] whenever any strategy in collection has dynamic config
+      def dynamic?
+        any?(&:dynamic?)
+      end
+
+      # @return [Boolean] whenever job is throttled or not
+      # by any strategy in collection.
+      def throttled?(*args)
+        any? { |s| s.throttled?(*args) }
+      end
+
+      # Marks job as being processed.
+      # @return [void]
+      def finalize!(*args)
+        each { |c| c.finalize!(*args) }
+      end
+
+      # Resets count of jobs of all avaliable strategies
+      # @return [void]
+      def reset!
+        each(&:reset!)
+      end
+
+      private
+
+      # @return [Base, nil]
+      def make_strategy(strategy, name, key_suffix, options)
+        return unless options
+
+        strategy.new("throttled:#{name}", {
+          :key_suffix => key_suffix,
+          **options
+        })
+      end
+    end
+  end
+end

data/lib/sidekiq/throttled/utils.rb

@@ -9,7 +9,7 @@ module Sidekiq
       # @param name [#to_s] Constant name
       # @return [Object, nil] Resolved constant or nil if failed.
       def constantize(name)
-        name.to_s.sub(/^::/, "").split("::").inject(Object, &:const_get)
+        name.to_s.sub(%r{^::}, "").split("::").inject(Object, &:const_get)
       rescue NameError
         Sidekiq.logger.warn { "Failed to constantize: #{name}" }
         nil

data/lib/sidekiq/throttled/version.rb

@@ -3,6 +3,6 @@
 module Sidekiq
   module Throttled
     # Gem version
-    VERSION = "0.10.0"
+    VERSION = "0.14.1"
   end
 end

data/lib/sidekiq/throttled/web/stats.rb

@@ -14,9 +14,7 @@ module Sidekiq
 
         # @param [Strategy::Concurrency, Strategy::Threshold] strategy
         def initialize(strategy)
-          if strategy&.dynamic?
-            raise ArgumentError, "Can't handle dynamic strategies"
-          end
+          raise ArgumentError, "Can't handle dynamic strategies" if strategy&.dynamic?
 
           @strategy = strategy
         end
@@ -27,9 +25,7 @@ module Sidekiq
 
           html = humanize_integer(@strategy.limit) << " jobs"
 
-          if @strategy.respond_to? :period
-            html << " per " << humanize_duration(@strategy.period)
-          end
+          html << " per " << humanize_duration(@strategy.period) if @strategy.respond_to?(:period)
 
           html << "<br />" << colorize_count(@strategy.count, @strategy.limit)
         end

data/lib/sidekiq/throttled/web/summary_fix.rb

@@ -4,14 +4,15 @@ module Sidekiq
   module Throttled
     module Web
       module SummaryFix
-        JAVASCRIPT = [File.read(__FILE__.sub(/\.rb$/, ".js")).freeze].freeze
+        JAVASCRIPT = [File.read(File.expand_path("summary_fix.js", __dir__)).freeze].freeze
         HEADERS    = { "Content-Type" => "application/javascript" }.freeze
 
         class << self
           attr_accessor :enabled
 
           def apply!(app)
-            Sidekiq::WebAction.send(:prepend, SummaryFix)
+            Sidekiq::WebAction.prepend SummaryFix
+
             app.get("/throttled/summary_fix") do
               [200, HEADERS.dup, JAVASCRIPT.dup]
             end

data/lib/sidekiq/throttled/web/throttled.html.erb

@@ -14,10 +14,14 @@
       <tr>
         <td style="vertical-align:middle;"><%= name %></td>
         <td style="vertical-align:middle;text-align:center;">
-          <%= Sidekiq::Throttled::Web::Stats.new(strategy.concurrency).to_html %>
+          <% strategy.concurrency.each do |concurrency| %>
+            <%= Sidekiq::Throttled::Web::Stats.new(concurrency).to_html %>
+          <% end %>
         </td>
         <td style="vertical-align:middle;text-align:center;">
-          <%= Sidekiq::Throttled::Web::Stats.new(strategy.threshold).to_html %>
+          <% strategy.threshold.each do |threshold| %>
+            <%= Sidekiq::Throttled::Web::Stats.new(threshold).to_html %>
+          <% end %>
         </td>
         <td style="vertical-align:middle;text-align:center;">
           <form action="<%= root_path %>throttled/<%= CGI.escape name %>/reset" method="post">

data/lib/sidekiq/throttled/web.rb

@@ -57,7 +57,7 @@ module Sidekiq
         end
 
         # rubocop:disable Metrics/AbcSize
-        def register_enhanced_queues_tab(app)
+        def register_enhanced_queues_tab(app) # rubocop:disable Metrics/MethodLength
           pauser = QueuesPauser.instance
 
           app.get("/enhanced-queues") do

data/lib/sidekiq/throttled/worker.rb

@@ -77,7 +77,7 @@ module Sidekiq
           Registry.add(self, **kwargs)
         end
 
-        # Adds current worker to preconfigured throtttling strtegy. Allows
+        # Adds current worker to preconfigured throttling strategy. Allows
         # sharing same pool for multiple workers.
         #
         # First of all we need to create shared throttling strategy:
@@ -85,7 +85,7 @@ module Sidekiq
         #     # Create google_api throttling strategy
         #     Sidekiq::Throttled::Registry.add(:google_api, {
         #       :threshold => { :limit => 123, :period => 1.hour },
-        #       :concurrency => { :limit => 123 }
+        #       :concurrency => { :limit => 10 }
         #     })
         #
         # Now we can assign it to our workers: