sidekiq-throttled 0.15.0 → 0.17.0

data/lib/sidekiq/throttled/fetch.rb

@@ -12,29 +12,6 @@ module Sidekiq
     #
     # @private
     class Fetch
-      module BulkRequeue
-        # Requeues all given units as a single operation.
-        #
-        # @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
-        # @param [Array<Fetch::UnitOfWork>] units
-        # @return [void]
-        def bulk_requeue(units, _options)
-          return if units.empty?
-
-          Sidekiq.logger.debug { "Re-queueing terminated jobs" }
-          Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
-          Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
-        rescue => e
-          Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
-        end
-      end
-
-      # https://github.com/mperham/sidekiq/commit/fce05c9d4b4c0411c982078a4cf3a63f20f739bc
-      if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.1.0")
-        extend BulkRequeue
-      else
-        include BulkRequeue
-      end
       # Timeout to sleep between fetch retries in case of no job received,
       # as well as timeout to wait for redis to give us something to work.
       TIMEOUT = 2
@@ -73,6 +50,20 @@ module Sidekiq
         nil
       end
 
+      def bulk_requeue(units, _options)
+        return if units.empty?
+
+        Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+        Sidekiq.redis do |conn|
+          conn.pipelined do |pipeline|
+            units.each { |unit| unit.requeue(pipeline) }
+          end
+        end
+        Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
+      rescue => e
+        Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
+      end
+
       private
 
       # Tries to pop pair of `queue` and job `message` out of sidekiq queues.
@@ -87,7 +78,7 @@ module Sidekiq
           return
         end
 
-        Sidekiq.redis { |conn| conn.brpop(*queues, TIMEOUT) }
+        Sidekiq.redis { |conn| conn.brpop(*queues, :timeout => TIMEOUT) }
       end
 
       # Returns list of queues to try to fetch jobs from.

data/lib/sidekiq/throttled/job.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+# internal
+require "sidekiq/throttled/registry"
+
+module Sidekiq
+  module Throttled
+    # Adds helpers to your worker classes
+    #
+    # @example Usage
+    #
+    #     class MyJob
+    #       include Sidekiq::Job
+    #       include Sidekiq::Throttled::Job
+    #
+    #       sidkiq_options :queue => :my_queue
+    #       sidekiq_throttle :threshold => { :limit => 123, :period => 1.hour }
+    #
+    #       def perform
+    #         # ...
+    #       end
+    #     end
+    #
+    # @see ClassMethods
+    module Job
+      # Extends worker class with {ClassMethods}.
+      #
+      # @note Using `included` hook with extending worker with {ClassMethods}
+      #   in order to make API inline with `include Sidekiq::Job`.
+      #
+      # @private
+      def self.included(worker)
+        worker.send(:extend, ClassMethods)
+      end
+
+      # Helper methods added to the singleton class of destination
+      module ClassMethods
+        # Registers some strategy for the worker.
+        #
+        # @example Allow max 123 MyJob jobs per hour
+        #
+        #     class MyJob
+        #       include Sidekiq::Job
+        #       include Sidekiq::Throttled::Job
+        #
+        #       sidekiq_throttle({
+        #         :threshold => { :limit => 123, :period => 1.hour }
+        #       })
+        #     end
+        #
+        # @example Allow max 10 concurrently running MyJob jobs
+        #
+        #     class MyJob
+        #       include Sidekiq::Job
+        #       include Sidekiq::Throttled::Job
+        #
+        #       sidekiq_throttle({
+        #         :concurrency => { :limit => 10 }
+        #       })
+        #     end
+        #
+        # @example Allow max 10 concurrent MyJob jobs and max 123 per hour
+        #
+        #     class MyJob
+        #       include Sidekiq::Job
+        #       include Sidekiq::Throttled::Job
+        #
+        #       sidekiq_throttle({
+        #         :threshold => { :limit => 123, :period => 1.hour },
+        #         :concurrency => { :limit => 10 }
+        #       })
+        #     end
+        #
+        # @see Registry.add
+        # @return [void]
+        def sidekiq_throttle(**kwargs)
+          Registry.add(self, **kwargs)
+        end
+
+        # Adds current worker to preconfigured throttling strategy. Allows
+        # sharing same pool for multiple workers.
+        #
+        # First of all we need to create shared throttling strategy:
+        #
+        #     # Create google_api throttling strategy
+        #     Sidekiq::Throttled::Registry.add(:google_api, {
+        #       :threshold => { :limit => 123, :period => 1.hour },
+        #       :concurrency => { :limit => 10 }
+        #     })
+        #
+        # Now we can assign it to our workers:
+        #
+        #     class FetchProfileJob
+        #       include Sidekiq::Job
+        #       include Sidekiq::Throttled::Job
+        #
+        #       sidekiq_throttle_as :google_api
+        #     end
+        #
+        #     class FetchCommentsJob
+        #       include Sidekiq::Job
+        #       include Sidekiq::Throttled::Job
+        #
+        #       sidekiq_throttle_as :google_api
+        #     end
+        #
+        # With the above configuration we ensure that there are maximum 10
+        # concurrently running jobs of FetchProfileJob or FetchCommentsJob
+        # allowed. And only 123 jobs of those are executed per hour.
+        #
+        # In other words, it will allow:
+        #
+        # - only `X` concurrent `FetchProfileJob`s
+        # - max `XX` `FetchProfileJob` per hour
+        # - only `Y` concurrent `FetchCommentsJob`s
+        # - max `YY` `FetchCommentsJob` per hour
+        #
+        # Where `(X + Y) == 10` and `(XX + YY) == 123`
+        #
+        # @see Registry.add_alias
+        # @return [void]
+        def sidekiq_throttle_as(name)
+          Registry.add_alias(self, name)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/throttled/strategy/base.rb

@@ -12,8 +12,12 @@ module Sidekiq
 
         def key(job_args)
           key = @base_key.dup
-          key << ":#{@key_suffix.call(*job_args)}" if @key_suffix
-          key
+          return key unless @key_suffix
+
+          key << ":#{@key_suffix.call(*job_args)}"
+        rescue => e
+          Sidekiq.logger.error "Failed to get key suffix: #{e}"
+          raise e
         end
       end
     end

data/lib/sidekiq/throttled/version.rb

@@ -3,6 +3,6 @@
 module Sidekiq
   module Throttled
     # Gem version
-    VERSION = "0.15.0"
+    VERSION = "0.17.0"
   end
 end

data/lib/sidekiq/throttled/web/stats.rb

@@ -37,7 +37,8 @@ module Sidekiq
           percentile = 100.00 * int / max
           lvl = if    80 <= percentile then "danger"
                 elsif 60 <= percentile then "warning"
-                else                        "success"
+                else
+                  "success"
                 end
 
           %(<span class="label label-#{lvl}">#{int}</span>)
@@ -61,10 +62,10 @@ module Sidekiq
 
         # @return [String]
         def humanize_integer(int)
-          digits = int.to_s.split ""
-          str    = digits.shift(digits.count % 3).join("")
+          digits = int.to_s.chars
+          str    = digits.shift(digits.count % 3).join
 
-          str << " " << digits.shift(3).join("") while digits.count.positive?
+          str << " " << digits.shift(3).join while digits.count.positive?
 
           str.strip
         end

data/lib/sidekiq/throttled/worker.rb

@@ -1,128 +1,13 @@
 # frozen_string_literal: true
 
-# internal
-require "sidekiq/throttled/registry"
+require "sidekiq/throttled/job"
 
 module Sidekiq
   module Throttled
-    # Adds helpers to your worker classes
-    #
-    # @example Usage
-    #
-    #     class MyWorker
-    #       include Sidekiq::Worker
-    #       include Sidekiq::Throttled::Worker
-    #
-    #       sidkiq_options :queue => :my_queue
-    #       sidekiq_throttle :threshold => { :limit => 123, :period => 1.hour }
-    #
-    #       def perform
-    #         # ...
-    #       end
-    #     end
-    #
-    # @see ClassMethods
-    module Worker
-      # Extends worker class with {ClassMethods}.
-      #
-      # @note Using `included` hook with extending worker with {ClassMethods}
-      #   in order to make API inline with `include Sidekiq::Worker`.
-      #
-      # @private
-      def self.included(worker)
-        worker.send(:extend, ClassMethods)
-      end
-
-      # Helper methods added to the singleton class of destination
-      module ClassMethods
-        # Registers some strategy for the worker.
-        #
-        # @example Allow max 123 MyWorker jobs per hour
-        #
-        #     class MyWorker
-        #       include Sidekiq::Worker
-        #       include Sidekiq::Throttled::Worker
-        #
-        #       sidekiq_throttle({
-        #         :threshold => { :limit => 123, :period => 1.hour }
-        #       })
-        #     end
-        #
-        # @example Allow max 10 concurrently running MyWorker jobs
-        #
-        #     class MyWorker
-        #       include Sidekiq::Worker
-        #       include Sidekiq::Throttled::Worker
-        #
-        #       sidekiq_throttle({
-        #         :concurrency => { :limit => 10 }
-        #       })
-        #     end
-        #
-        # @example Allow max 10 concurrent MyWorker jobs and max 123 per hour
-        #
-        #     class MyWorker
-        #       include Sidekiq::Worker
-        #       include Sidekiq::Throttled::Worker
-        #
-        #       sidekiq_throttle({
-        #         :threshold => { :limit => 123, :period => 1.hour },
-        #         :concurrency => { :limit => 10 }
-        #       })
-        #     end
-        #
-        # @see Registry.add
-        # @return [void]
-        def sidekiq_throttle(**kwargs)
-          Registry.add(self, **kwargs)
-        end
-
-        # Adds current worker to preconfigured throttling strategy. Allows
-        # sharing same pool for multiple workers.
-        #
-        # First of all we need to create shared throttling strategy:
-        #
-        #     # Create google_api throttling strategy
-        #     Sidekiq::Throttled::Registry.add(:google_api, {
-        #       :threshold => { :limit => 123, :period => 1.hour },
-        #       :concurrency => { :limit => 10 }
-        #     })
-        #
-        # Now we can assign it to our workers:
-        #
-        #     class FetchProfileJob
-        #       include Sidekiq::Worker
-        #       include Sidekiq::Throttled::Worker
-        #
-        #       sidekiq_throttle_as :google_api
-        #     end
-        #
-        #     class FetchCommentsJob
-        #       include Sidekiq::Worker
-        #       include Sidekiq::Throttled::Worker
-        #
-        #       sidekiq_throttle_as :google_api
-        #     end
-        #
-        # With the above configuration we ensure that there are maximum 10
-        # concurrently running jobs of FetchProfileJob or FetchCommentsJob
-        # allowed. And only 123 jobs of those are executed per hour.
-        #
-        # In other words, it will allow:
-        #
-        # - only `X` concurrent `FetchProfileJob`s
-        # - max `XX` `FetchProfileJob` per hour
-        # - only `Y` concurrent `FetchCommentsJob`s
-        # - max `YY` `FetchCommentsJob` per hour
-        #
-        # Where `(X + Y) == 10` and `(XX + YY) == 123`
-        #
-        # @see Registry.add_alias
-        # @return [void]
-        def sidekiq_throttle_as(name)
-          Registry.add_alias(self, name)
-        end
-      end
-    end
+    # A new module, Sidekiq::Job, was added in Sidekiq version 6.3.0 as a
+    # simple alias for Sidekiq::Worker as the term "worker" was considered
+    # too generic and confusing. Many people call a Sidekiq process a "worker"
+    # whereas others call the thread that executes jobs a "worker".
+    Worker = Job
   end
 end

data/lib/sidekiq/throttled.rb

@@ -9,6 +9,7 @@ require "sidekiq/throttled/communicator"
 require "sidekiq/throttled/configuration"
 require "sidekiq/throttled/queues_pauser"
 require "sidekiq/throttled/registry"
+require "sidekiq/throttled/job"
 require "sidekiq/throttled/worker"
 require "sidekiq/throttled/utils"
 
@@ -21,12 +22,12 @@ module Sidekiq
   #     require "sidekiq/throttled"
   #     Sidekiq::Throttled.setup!
   #
-  # Once you've done that you can include {Sidekiq::Throttled::Worker} to your
+  # Once you've done that you can include {Sidekiq::Throttled::Job} to your
   # job classes and configure throttling:
   #
-  #     class MyWorker
-  #       include Sidekiq::Worker
-  #       include Sidekiq::Throttled::Worker
+  #     class MyJob
+  #       include Sidekiq::Job
+  #       include Sidekiq::Throttled::Job
   #
   #       sidekiq_options :queue => :my_queue
   #
@@ -61,7 +62,7 @@ module Sidekiq
         QueuesPauser.instance.setup!
 
         Sidekiq.configure_server do |config|
-          setup_strategy!
+          setup_strategy!(config)
 
           require "sidekiq/throttled/middleware"
           config.server_middleware do |chain|
@@ -76,7 +77,7 @@ module Sidekiq
       # @return [Boolean]
       def throttled?(message)
         message = JSON.parse message
-        job = message.fetch("class") { return false }
+        job = message.fetch("wrapped") { message.fetch("class") { return false } }
         jid = message.fetch("jid") { return false }
 
         preload_constant! job
@@ -93,16 +94,13 @@ module Sidekiq
       private
 
       # @return [void]
-      def setup_strategy!
+      def setup_strategy!(sidekiq_config)
         require "sidekiq/throttled/fetch"
 
-        # https://github.com/mperham/sidekiq/commit/fce05c9d4b4c0411c982078a4cf3a63f20f739bc
-        Sidekiq.options[:fetch] =
-          if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.1.0")
-            Sidekiq::Throttled::Fetch
-          else
-            Sidekiq::Throttled::Fetch.new(Sidekiq.options)
-          end
+        # https://github.com/mperham/sidekiq/commit/67daa7a408b214d593100f782271ed108686c147
+        sidekiq_config = sidekiq_config.options if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.5.0")
+
+        sidekiq_config[:fetch] = Sidekiq::Throttled::Fetch.new(sidekiq_config)
       end
 
       # Tries to preload constant by it's name once.

data/{.rubocop → rubocop}/rspec.yml

@@ -1,3 +1,3 @@
 RSpec/ExampleLength:
   Enabled: true
-  Max: 10
+  Max: 15

data/sidekiq-throttled.gemspec

@@ -8,27 +8,29 @@ require "sidekiq/throttled/version"
 Gem::Specification.new do |spec|
   spec.name          = "sidekiq-throttled"
   spec.version       = Sidekiq::Throttled::VERSION
-  spec.authors       = ["Alexey V Zapparov"]
-  spec.email         = ["ixti@member.fsf.org"]
+  spec.authors       = ["Alexey Zapparov"]
+  spec.email         = ["alexey@zapparov.com"]
 
   spec.summary       = "Concurrency and threshold throttling for Sidekiq."
   spec.description   = "Concurrency and threshold throttling for Sidekiq."
-  spec.homepage      = "https://github.com/sensortower/sidekiq-throttled"
+  spec.homepage      = "https://github.com/ixti/sidekiq-throttled"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match %r{^(test|spec|features)/}
   end
 
+  spec.metadata["rubygems_mfa_required"] = "true"
+
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = ">= 2.6"
+  spec.required_ruby_version = ">= 2.7"
 
   spec.add_runtime_dependency "concurrent-ruby"
   spec.add_runtime_dependency "redis-prescription"
-  spec.add_runtime_dependency "sidekiq"
+  spec.add_runtime_dependency "sidekiq", ">= 6.4"
 
   spec.add_development_dependency "bundler", ">= 2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-throttled
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.17.0
 platform: ruby
 authors:
-- Alexey V Zapparov
+- Alexey Zapparov
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-12-16 00:00:00.000000000 Z
+date: 2022-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6.4'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -68,22 +68,17 @@ dependencies:
         version: '2.0'
 description: Concurrency and threshold throttling for Sidekiq.
 email:
-- ixti@member.fsf.org
+- alexey@zapparov.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".coveralls.yml"
+- ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".rubocop/layout.yml"
-- ".rubocop/lint.yml"
-- ".rubocop/metrics.yml"
-- ".rubocop/performance.yml"
-- ".rubocop/rspec.yml"
-- ".rubocop/style.yml"
 - ".rubocop_todo.yml"
 - ".travis.yml"
 - ".yardopts"
@@ -94,22 +89,19 @@ files:
 - LICENSE.md
 - README.md
 - Rakefile
-- gemfiles/sidekiq_5.0.gemfile
-- gemfiles/sidekiq_5.1.gemfile
-- gemfiles/sidekiq_5.2.gemfile
-- gemfiles/sidekiq_6.0.gemfile
-- gemfiles/sidekiq_6.1.gemfile
-- gemfiles/sidekiq_6.2.gemfile
-- gemfiles/sidekiq_6.3.gemfile
+- gemfiles/sidekiq_6.4.gemfile
+- gemfiles/sidekiq_6.5.gemfile
 - lib/sidekiq/throttled.rb
 - lib/sidekiq/throttled/communicator.rb
 - lib/sidekiq/throttled/communicator/callbacks.rb
+- lib/sidekiq/throttled/communicator/exception_handler.rb
 - lib/sidekiq/throttled/communicator/listener.rb
 - lib/sidekiq/throttled/configuration.rb
 - lib/sidekiq/throttled/errors.rb
 - lib/sidekiq/throttled/expirable_list.rb
 - lib/sidekiq/throttled/fetch.rb
 - lib/sidekiq/throttled/fetch/unit_of_work.rb
+- lib/sidekiq/throttled/job.rb
 - lib/sidekiq/throttled/middleware.rb
 - lib/sidekiq/throttled/patches/queue.rb
 - lib/sidekiq/throttled/queue_name.rb
@@ -132,11 +124,18 @@ files:
 - lib/sidekiq/throttled/web/summary_fix.rb
 - lib/sidekiq/throttled/web/throttled.html.erb
 - lib/sidekiq/throttled/worker.rb
+- rubocop/layout.yml
+- rubocop/lint.yml
+- rubocop/metrics.yml
+- rubocop/performance.yml
+- rubocop/rspec.yml
+- rubocop/style.yml
 - sidekiq-throttled.gemspec
-homepage: https://github.com/sensortower/sidekiq-throttled
+homepage: https://github.com/ixti/sidekiq-throttled
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -145,14 +144,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Concurrency and threshold throttling for Sidekiq.

data/gemfiles/sidekiq_5.0.gemfile

@@ -1,31 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "appraisal"
-gem "rake"
-gem "rspec"
-gem "rubocop", "~> 0.90.0", require: false
-gem "rubocop-performance", "~> 1.8.0", require: false
-gem "rubocop-rspec", "~> 1.43.2", require: false
-gem "sidekiq", "~> 5.0.0"
-
-group :development do
-  gem "byebug"
-  gem "guard", require: false
-  gem "guard-rspec", require: false
-  gem "guard-rubocop", require: false
-end
-
-group :test do
-  gem "apparition"
-  gem "capybara"
-  gem "coveralls", require: false
-  gem "puma"
-  gem "rack-test"
-  gem "simplecov"
-  gem "sinatra"
-  gem "timecop"
-end
-
-gemspec path: "../"

data/gemfiles/sidekiq_6.0.gemfile

@@ -1,31 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "appraisal"
-gem "rake"
-gem "rspec"
-gem "rubocop", "~> 0.90.0", require: false
-gem "rubocop-performance", "~> 1.8.0", require: false
-gem "rubocop-rspec", "~> 1.43.2", require: false
-gem "sidekiq", "~> 6.0.0"
-
-group :development do
-  gem "byebug"
-  gem "guard", require: false
-  gem "guard-rspec", require: false
-  gem "guard-rubocop", require: false
-end
-
-group :test do
-  gem "apparition"
-  gem "capybara"
-  gem "coveralls", require: false
-  gem "puma"
-  gem "rack-test"
-  gem "simplecov"
-  gem "sinatra"
-  gem "timecop"
-end
-
-gemspec path: "../"