sidekiq-throttled 0.13.0 → 0.16.0

data/gemfiles/{sidekiq_5.1.gemfile → sidekiq_6.3.gemfile}

@@ -5,10 +5,7 @@ source "https://rubygems.org"
 gem "appraisal"
 gem "rake"
 gem "rspec"
-gem "rubocop", "~> 0.82.0", require: false
-gem "rubocop-performance", "~>1.5.2", require: false
-gem "rubocop-rspec", "~> 1.39.0", require: false
-gem "sidekiq", "~> 5.1.0"
+gem "sidekiq", "~> 6.3.0"
 
 group :development do
   gem "byebug"
@@ -20,12 +17,17 @@ end
 group :test do
   gem "apparition"
   gem "capybara"
-  gem "coveralls", require: false
   gem "puma"
   gem "rack-test"
-  gem "simplecov"
   gem "sinatra"
   gem "timecop"
 end
 
+group :lint do
+  gem "rubocop", require: false
+  gem "rubocop-performance", require: false
+  gem "rubocop-rake", require: false
+  gem "rubocop-rspec", require: false
+end
+
 gemspec path: "../"

data/gemfiles/{sidekiq_5.2.gemfile → sidekiq_6.4.gemfile}

@@ -5,10 +5,7 @@ source "https://rubygems.org"
 gem "appraisal"
 gem "rake"
 gem "rspec"
-gem "rubocop", "~> 0.82.0", require: false
-gem "rubocop-performance", "~>1.5.2", require: false
-gem "rubocop-rspec", "~> 1.39.0", require: false
-gem "sidekiq", "~> 5.2.0"
+gem "sidekiq", "~> 6.4.0"
 
 group :development do
   gem "byebug"
@@ -20,12 +17,17 @@ end
 group :test do
   gem "apparition"
   gem "capybara"
-  gem "coveralls", require: false
   gem "puma"
   gem "rack-test"
-  gem "simplecov"
   gem "sinatra"
   gem "timecop"
 end
 
+group :lint do
+  gem "rubocop", require: false
+  gem "rubocop-performance", require: false
+  gem "rubocop-rake", require: false
+  gem "rubocop-rspec", require: false
+end
+
 gemspec path: "../"

data/gemfiles/sidekiq_6.5.gemfile

@@ -0,0 +1,33 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "appraisal"
+gem "rake"
+gem "rspec"
+gem "sidekiq", "~> 6.5.0"
+
+group :development do
+  gem "byebug"
+  gem "guard", require: false
+  gem "guard-rspec", require: false
+  gem "guard-rubocop", require: false
+end
+
+group :test do
+  gem "apparition"
+  gem "capybara"
+  gem "puma"
+  gem "rack-test"
+  gem "sinatra"
+  gem "timecop"
+end
+
+group :lint do
+  gem "rubocop", require: false
+  gem "rubocop-performance", require: false
+  gem "rubocop-rake", require: false
+  gem "rubocop-rspec", require: false
+end
+
+gemspec path: "../"

data/lib/sidekiq/throttled/communicator/callbacks.rb

@@ -2,7 +2,7 @@
 
 require "fiber"
 
-require "sidekiq/exception_handler"
+require "sidekiq/throttled/communicator/exception_handler"
 
 module Sidekiq
   module Throttled
@@ -55,17 +55,15 @@ module Sidekiq
         # @return [void]
         def run(event, payload = nil)
           @mutex.synchronize do
-            Fiber.new do
+            fiber = Fiber.new do
               @handlers[event.to_s].each do |callback|
-                begin
-                  callback.call(payload)
-                rescue => e
-                  handle_exception(e, {
-                    :context => "sidekiq:throttled"
-                  })
-                end
+                callback.call(payload)
+              rescue => e
+                handle_exception(e, :context => "sidekiq:throttled")
               end
-            end.resume
+            end
+
+            fiber.resume
           end
         end
       end

data/lib/sidekiq/throttled/communicator/exception_handler.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+require "sidekiq/version"
+
+module Sidekiq
+  module Throttled
+    class Communicator
+      if Sidekiq::VERSION >= "6.5.0"
+        module ExceptionHandler
+          def handle_exception(*args)
+            Sidekiq.handle_exception(*args)
+          end
+        end
+
+        # NOTE: `Sidekiq.default_error_handler` is private API
+        Sidekiq.error_handlers << Sidekiq.method(:default_error_handler)
+      else
+        require "sidekiq/exception_handler"
+
+        ExceptionHandler = ::Sidekiq::ExceptionHandler
+      end
+    end
+  end
+end

data/lib/sidekiq/throttled/communicator/listener.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "sidekiq/exception_handler"
+require "sidekiq/throttled/communicator/exception_handler"
 
 module Sidekiq
   module Throttled
@@ -62,7 +62,7 @@ module Sidekiq
         # - `Exception` is recorded to the log and re-raised.
         #
         # @return [void]
-        def listen
+        def listen # rubocop:disable Metrics/MethodLength
           subscribe
         rescue Sidekiq::Shutdown
           @terminated = true
@@ -88,7 +88,7 @@ module Sidekiq
         # @see http://redis.io/commands/subscribe
         # @see Callbacks#run
         # @return [void]
-        def subscribe
+        def subscribe # rubocop:disable Metrics/MethodLength
           Sidekiq.redis do |conn|
             conn.subscribe @channel do |on|
               on.subscribe do
@@ -97,7 +97,7 @@ module Sidekiq
               end
 
               on.message do |_channel, data|
-                message, payload = Marshal.load(data)
+                message, payload = Marshal.load(data) # rubocop:disable Security/MarshalLoad:
                 @callbacks.run("message:#{message}", payload)
               end
             end

data/lib/sidekiq/throttled/communicator.rb

@@ -2,7 +2,7 @@
 
 require "singleton"
 
-require "sidekiq/exception_handler"
+require "sidekiq/throttled/communicator/exception_handler"
 require "sidekiq/throttled/communicator/listener"
 require "sidekiq/throttled/communicator/callbacks"
 

data/lib/sidekiq/throttled/expirable_list.rb

@@ -2,8 +2,6 @@
 
 require "monitor"
 
-require "concurrent/utility/monotonic_time"
-
 module Sidekiq
   module Throttled
     # List that tracks when elements were added and enumerates over those not
@@ -24,7 +22,6 @@ module Sidekiq
     # It does not deduplicates elements. Eviction happens only upon elements
     # retrieval (see {#each}).
     #
-    # @see http://ruby-concurrency.github.io/concurrent-ruby/Concurrent.html#monotonic_time-class_method
     # @see https://ruby-doc.org/core/Process.html#method-c-clock_gettime
     # @see https://linux.die.net/man/3/clock_gettime
     #
@@ -44,7 +41,7 @@ module Sidekiq
       # @params element [Object]
       # @return [ExpirableList] self
       def <<(element)
-        @mon.synchronize { @arr << [Concurrent.monotonic_time, element] }
+        @mon.synchronize { @arr << [::Process.clock_gettime(::Process::CLOCK_MONOTONIC), element] }
         self
       end
 
@@ -58,7 +55,7 @@ module Sidekiq
         return to_enum __method__ unless block_given?
 
         @mon.synchronize do
-          horizon = Concurrent.monotonic_time - @ttl
+          horizon = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @ttl
 
           # drop all elements older than horizon
           @arr.shift while @arr[0] && @arr[0][0] < horizon

data/lib/sidekiq/throttled/fetch/unit_of_work.rb

@@ -49,9 +49,14 @@ module Sidekiq
         #   process was terminated. It is a reverse of whatever fetcher was
         #   doing to pull the job out of queue.
         #
+        # @param [Redis] pipelined connection for requeing via Redis#pipelined
         # @return [void]
-        def requeue
-          Sidekiq.redis { |conn| conn.rpush(QueueName.expand(queue_name), job) }
+        def requeue(pipeline = nil)
+          if pipeline
+            pipeline.rpush(QueueName.expand(queue_name), job)
+          else
+            Sidekiq.redis { |conn| conn.rpush(QueueName.expand(queue_name), job) }
+          end
         end
 
         # Pushes job back to the head of the queue, so that job won't be tried

data/lib/sidekiq/throttled/fetch.rb

@@ -22,7 +22,11 @@ module Sidekiq
           return if units.empty?
 
           Sidekiq.logger.debug { "Re-queueing terminated jobs" }
-          Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
+          Sidekiq.redis do |conn|
+            conn.pipelined do |pipeline|
+              units.each { |unit| unit.requeue(pipeline) }
+            end
+          end
           Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
         rescue => e
           Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")

data/lib/sidekiq/throttled/job.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+# internal
+require "sidekiq/throttled/registry"
+
+module Sidekiq
+  module Throttled
+    # Adds helpers to your worker classes
+    #
+    # @example Usage
+    #
+    #     class MyWorker
+    #       include Sidekiq::Worker
+    #       include Sidekiq::Throttled::Worker
+    #
+    #       sidkiq_options :queue => :my_queue
+    #       sidekiq_throttle :threshold => { :limit => 123, :period => 1.hour }
+    #
+    #       def perform
+    #         # ...
+    #       end
+    #     end
+    #
+    # @see ClassMethods
+    module Job
+      # Extends worker class with {ClassMethods}.
+      #
+      # @note Using `included` hook with extending worker with {ClassMethods}
+      #   in order to make API inline with `include Sidekiq::Worker`.
+      #
+      # @private
+      def self.included(worker)
+        worker.send(:extend, ClassMethods)
+      end
+
+      # Helper methods added to the singleton class of destination
+      module ClassMethods
+        # Registers some strategy for the worker.
+        #
+        # @example Allow max 123 MyWorker jobs per hour
+        #
+        #     class MyWorker
+        #       include Sidekiq::Worker
+        #       include Sidekiq::Throttled::Worker
+        #
+        #       sidekiq_throttle({
+        #         :threshold => { :limit => 123, :period => 1.hour }
+        #       })
+        #     end
+        #
+        # @example Allow max 10 concurrently running MyWorker jobs
+        #
+        #     class MyWorker
+        #       include Sidekiq::Worker
+        #       include Sidekiq::Throttled::Worker
+        #
+        #       sidekiq_throttle({
+        #         :concurrency => { :limit => 10 }
+        #       })
+        #     end
+        #
+        # @example Allow max 10 concurrent MyWorker jobs and max 123 per hour
+        #
+        #     class MyWorker
+        #       include Sidekiq::Worker
+        #       include Sidekiq::Throttled::Worker
+        #
+        #       sidekiq_throttle({
+        #         :threshold => { :limit => 123, :period => 1.hour },
+        #         :concurrency => { :limit => 10 }
+        #       })
+        #     end
+        #
+        # @see Registry.add
+        # @return [void]
+        def sidekiq_throttle(**kwargs)
+          Registry.add(self, **kwargs)
+        end
+
+        # Adds current worker to preconfigured throttling strategy. Allows
+        # sharing same pool for multiple workers.
+        #
+        # First of all we need to create shared throttling strategy:
+        #
+        #     # Create google_api throttling strategy
+        #     Sidekiq::Throttled::Registry.add(:google_api, {
+        #       :threshold => { :limit => 123, :period => 1.hour },
+        #       :concurrency => { :limit => 10 }
+        #     })
+        #
+        # Now we can assign it to our workers:
+        #
+        #     class FetchProfileJob
+        #       include Sidekiq::Worker
+        #       include Sidekiq::Throttled::Worker
+        #
+        #       sidekiq_throttle_as :google_api
+        #     end
+        #
+        #     class FetchCommentsJob
+        #       include Sidekiq::Worker
+        #       include Sidekiq::Throttled::Worker
+        #
+        #       sidekiq_throttle_as :google_api
+        #     end
+        #
+        # With the above configuration we ensure that there are maximum 10
+        # concurrently running jobs of FetchProfileJob or FetchCommentsJob
+        # allowed. And only 123 jobs of those are executed per hour.
+        #
+        # In other words, it will allow:
+        #
+        # - only `X` concurrent `FetchProfileJob`s
+        # - max `XX` `FetchProfileJob` per hour
+        # - only `Y` concurrent `FetchCommentsJob`s
+        # - max `YY` `FetchCommentsJob` per hour
+        #
+        # Where `(X + Y) == 10` and `(XX + YY) == 123`
+        #
+        # @see Registry.add_alias
+        # @return [void]
+        def sidekiq_throttle_as(name)
+          Registry.add_alias(self, name)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/throttled/queue_name.rb

@@ -7,7 +7,7 @@ module Sidekiq
     # @private
     module QueueName
       # RegExp used to stip out any redisr-namespace prefixes with `queue:`.
-      QUEUE_NAME_PREFIX_RE = /.*queue:/.freeze
+      QUEUE_NAME_PREFIX_RE = %r{.*queue:}.freeze
       private_constant :QUEUE_NAME_PREFIX_RE
 
       class << self

data/lib/sidekiq/throttled/registry.rb

@@ -18,20 +18,16 @@ module Sidekiq
 
         # Adds strategy to the registry.
         #
-        # @note prints a warning to STDERR upon duplicate strategy name
         # @param (see Strategy#initialize)
         # @return [Strategy]
         def add(name, **kwargs)
           name = name.to_s
 
-          warn "Duplicate strategy name: #{name}" if @strategies[name]
-
           @strategies[name] = Strategy.new(name, **kwargs)
         end
 
         # Adds alias for existing strategy.
         #
-        # @note prints a warning to STDERR upon duplicate strategy name
         # @param (#to_s) new_name
         # @param (#to_s) old_name
         # @raise [RuntimeError] if no strategy found with `old_name`
@@ -40,7 +36,6 @@ module Sidekiq
           new_name = new_name.to_s
           old_name = old_name.to_s
 
-          warn "Duplicate strategy name: #{new_name}" if @strategies[new_name]
           raise "Strategy not found: #{old_name}" unless @strategies[old_name]
 
           @aliases[new_name] = @strategies[old_name]

data/lib/sidekiq/throttled/strategy.rb

@@ -2,6 +2,7 @@
 
 # internal
 require "sidekiq/throttled/errors"
+require "sidekiq/throttled/strategy_collection"
 require "sidekiq/throttled/strategy/concurrency"
 require "sidekiq/throttled/strategy/threshold"
 
@@ -30,15 +31,20 @@ module Sidekiq
       #   See keyword args of {Strategy::Threshold#initialize} for details.
       # @param [#call] key_suffix Dynamic key suffix generator.
       # @param [#call] observer Process called after throttled.
-      def initialize(
-        name,
-        concurrency: nil, threshold: nil, key_suffix: nil, observer: nil
-      )
-        @observer    = observer
-        @concurrency = make_strategy(Concurrency, name, key_suffix, concurrency)
-        @threshold   = make_strategy(Threshold, name, key_suffix, threshold)
+      def initialize(name, concurrency: nil, threshold: nil, key_suffix: nil, observer: nil) # rubocop:disable Metrics/MethodLength
+        @observer = observer
 
-        return if @concurrency || @threshold
+        @concurrency = StrategyCollection.new(concurrency,
+          :strategy   => Concurrency,
+          :name       => name,
+          :key_suffix => key_suffix)
+
+        @threshold = StrategyCollection.new(threshold,
+          :strategy   => Threshold,
+          :name       => name,
+          :key_suffix => key_suffix)
+
+        return if @concurrency.any? || @threshold.any?
 
         raise ArgumentError, "Neither :concurrency nor :threshold given"
       end
@@ -60,6 +66,7 @@ module Sidekiq
 
         if @threshold&.throttled?(*job_args)
           @observer&.call(:threshold, *job_args)
+
           finalize!(jid, *job_args)
           return true
         end
@@ -79,15 +86,6 @@ module Sidekiq
         @concurrency&.reset!
         @threshold&.reset!
       end
-
-      private
-
-      # @return [Base, nil]
-      def make_strategy(strategy, name, key_suffix, options)
-        return unless options
-
-        strategy.new("throttled:#{name}", :key_suffix => key_suffix, **options)
-      end
     end
   end
 end

data/lib/sidekiq/throttled/strategy_collection.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# internal
+module Sidekiq
+  module Throttled
+    # Collection which transparently group several meta-strategies of one kind
+    #
+    # @private
+    class StrategyCollection
+      include Enumerable
+
+      attr_reader :strategies
+
+      # @param [Hash, Array, nil] strategies Concurrency or Threshold options
+      #   or array of options.
+      #   See keyword args of {Strategy::Concurrency#initialize} for details.
+      #   See keyword args of {Strategy::Threshold#initialize} for details.
+      # @param [Class] strategy class of strategy: Concurrency or Threshold
+      # @param [#to_s] name
+      # @param [#call] key_suffix Dynamic key suffix generator.
+      def initialize(strategies, strategy:, name:, key_suffix:)
+        strategies = (strategies.is_a?(Hash) ? [strategies] : Array(strategies))
+        @strategies = strategies.map do |options|
+          make_strategy(strategy, name, key_suffix, options)
+        end
+      end
+
+      # @param [#call] block
+      # Iterates each strategy in collection
+      def each(&block)
+        @strategies.each(&block)
+      end
+
+      # @return [Boolean] whenever any strategy in collection has dynamic config
+      def dynamic?
+        any?(&:dynamic?)
+      end
+
+      # @return [Boolean] whenever job is throttled or not
+      # by any strategy in collection.
+      def throttled?(*args)
+        any? { |s| s.throttled?(*args) }
+      end
+
+      # Marks job as being processed.
+      # @return [void]
+      def finalize!(*args)
+        each { |c| c.finalize!(*args) }
+      end
+
+      # Resets count of jobs of all avaliable strategies
+      # @return [void]
+      def reset!
+        each(&:reset!)
+      end
+
+      private
+
+      # @return [Base, nil]
+      def make_strategy(strategy, name, key_suffix, options)
+        return unless options
+
+        strategy.new("throttled:#{name}",
+          :key_suffix => key_suffix,
+          **options)
+      end
+    end
+  end
+end

data/lib/sidekiq/throttled/utils.rb

@@ -9,7 +9,7 @@ module Sidekiq
       # @param name [#to_s] Constant name
       # @return [Object, nil] Resolved constant or nil if failed.
       def constantize(name)
-        name.to_s.sub(/^::/, "").split("::").inject(Object, &:const_get)
+        name.to_s.sub(%r{^::}, "").split("::").inject(Object, &:const_get)
       rescue NameError
         Sidekiq.logger.warn { "Failed to constantize: #{name}" }
         nil

data/lib/sidekiq/throttled/version.rb

@@ -3,6 +3,6 @@
 module Sidekiq
   module Throttled
     # Gem version
-    VERSION = "0.13.0"
+    VERSION = "0.16.0"
   end
 end

data/lib/sidekiq/throttled/web/stats.rb

@@ -37,7 +37,8 @@ module Sidekiq
           percentile = 100.00 * int / max
           lvl = if    80 <= percentile then "danger"
                 elsif 60 <= percentile then "warning"
-                else                        "success"
+                else
+                  "success"
                 end
 
           %(<span class="label label-#{lvl}">#{int}</span>)
@@ -61,10 +62,10 @@ module Sidekiq
 
         # @return [String]
         def humanize_integer(int)
-          digits = int.to_s.split ""
-          str    = digits.shift(digits.count % 3).join("")
+          digits = int.to_s.chars
+          str    = digits.shift(digits.count % 3).join
 
-          str << " " << digits.shift(3).join("") while digits.count.positive?
+          str << " " << digits.shift(3).join while digits.count.positive?
 
           str.strip
         end

data/lib/sidekiq/throttled/web/summary_fix.rb

@@ -4,7 +4,7 @@ module Sidekiq
   module Throttled
     module Web
       module SummaryFix
-        JAVASCRIPT = [File.read(__FILE__.sub(/\.rb$/, ".js")).freeze].freeze
+        JAVASCRIPT = [File.read(File.expand_path("summary_fix.js", __dir__)).freeze].freeze
         HEADERS    = { "Content-Type" => "application/javascript" }.freeze
 
         class << self

data/lib/sidekiq/throttled/web/throttled.html.erb

@@ -14,10 +14,14 @@
       <tr>
         <td style="vertical-align:middle;"><%= name %></td>
         <td style="vertical-align:middle;text-align:center;">
-          <%= Sidekiq::Throttled::Web::Stats.new(strategy.concurrency).to_html %>
+          <% strategy.concurrency.each do |concurrency| %>
+            <%= Sidekiq::Throttled::Web::Stats.new(concurrency).to_html %>
+          <% end %>
         </td>
         <td style="vertical-align:middle;text-align:center;">
-          <%= Sidekiq::Throttled::Web::Stats.new(strategy.threshold).to_html %>
+          <% strategy.threshold.each do |threshold| %>
+            <%= Sidekiq::Throttled::Web::Stats.new(threshold).to_html %>
+          <% end %>
         </td>
         <td style="vertical-align:middle;text-align:center;">
           <form action="<%= root_path %>throttled/<%= CGI.escape name %>/reset" method="post">

data/lib/sidekiq/throttled/web.rb

@@ -57,7 +57,7 @@ module Sidekiq
         end
 
         # rubocop:disable Metrics/AbcSize
-        def register_enhanced_queues_tab(app)
+        def register_enhanced_queues_tab(app) # rubocop:disable Metrics/MethodLength
           pauser = QueuesPauser.instance
 
           app.get("/enhanced-queues") do