sidekiq-throttled 0.10.0.beta → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6c32e77b9ca7aa1dd0bdad2b74c6fa795f4f660b5d7e7a8f92b16f6b419112e
-  data.tar.gz: 8d8b4ca63ae59d184846e297fc6281f3b67f282b4f546d1ada0a002e84f65232
+  metadata.gz: a204bc340965b64fe16f9b896815c9b3fbf383418b0774871d90c35d4c8fb061
+  data.tar.gz: bb2b9681a7caf33d3418ab2f475bc0ab2f6141ed9e6609d55d680815da71c4e6
 SHA512:
-  metadata.gz: 24159823a56f6765079ebb27b16b2a2288950a4bab13f628f2d48bd45939400c29d2c640db5af69dd2320e5b973d0a5afeda15cceb457fe90db1b69f8f81fa53
-  data.tar.gz: 60c65d079962a86fdd02787d1ff2262ace7a31c16a751835038f9d06031f5af8d3d1bb65ec2bfd97ccf542a83cfa97da0ea2a1a70b8993c6793c8bf50fb2ab64
+  metadata.gz: '0801102f967a02d3a023b9d8bfe04b8d44280c5f2986182ef4a5cc6e971091afcb3fb1b6c845f382829de071e11d6a3111cb04a21b040b022b61ecac76d5a422'
+  data.tar.gz: 243ceeb9a791417edc07a7f697660035e52a61f3db4aefbe93508be5037d1af1932d2b3ad81886ee1b9ef8408e97f1bef8cac03b9a78d8e4c29a2197f9f927f6

data/.rubocop.yml

@@ -8,7 +8,7 @@ require:
 
 AllCops:
   DisplayCopNames: true
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.4
   Exclude:
     - "gemfiles/*"
 

data/.travis.yml

@@ -9,8 +9,9 @@ cache: bundler
 before_install:
   - gem update --system
   - gem --version
-  - gem install bundler --no-rdoc --no-ri
+  - gem install bundler
   - bundle --version
+  # Install pahantomjs
   - mkdir travis-phantomjs
   - wget https://s3.amazonaws.com/travis-phantomjs/phantomjs-2.0.0-ubuntu-12.04.tar.bz2 -O $PWD/travis-phantomjs/phantomjs-2.0.0-ubuntu-12.04.tar.bz2
   - tar -xvf $PWD/travis-phantomjs/phantomjs-2.0.0-ubuntu-12.04.tar.bz2 -C $PWD/travis-phantomjs
@@ -19,15 +20,15 @@ before_install:
 install: bundle install --without development
 
 rvm:
-  - 2.3
   - 2.4
   - 2.5
+  - 2.6
 
 matrix:
   fast_finish: true
   include:
     -
-      rvm: 2.3
+      rvm: 2.4
       env: SUITE="rubocop"
       gemfile: Gemfile
 

data/CHANGES.md

@@ -1,3 +1,18 @@
+## 0.10.0 (2019-06-22)
+
+* [#60](https://github.com/sensortower/sidekiq-throttled/pull/60)
+  Skip throttling check in redis if limit is 0.
+  ([@mstruve])
+
+* [#58](https://github.com/sensortower/sidekiq-throttled/pull/58)
+  Improve documentation bout TTL.
+  ([@ziaulrehman40])
+
+* Improve reliability of paused queues, by resyncing list of paused queues
+  on schedule.
+  ([@ixti])
+
+
 ## 0.9.0 (2018-09-11)
 
 * Add support of Sidekiq 5.2.x
@@ -176,3 +191,5 @@
 [@palanglung]: https://github.com/palanglung
 [@azach]: https://github.com/azach
 [@iporsut]: https://github.com/iporsut
+[@mstruve]: https://github.com/mstruve
+[@ziaulrehman40]: https://github.com/ziaulrehman40

data/README.md

@@ -117,6 +117,25 @@ if you are using dynamic limit/period options. Otherwise you risk getting into
 some trouble.
 
 
+### Concurrency throttling fine-tuning
+
+Concurrency throttling is based on distributed locks. Those locks have default
+time to live (TTL) set to 15 minutes. If your job takes more than 15 minutes
+to finish, lock will be released and you might end up with more jobs running
+concurrently than you expect.
+
+This is done to avoid deadlocks - when by any reason (e.g. Sidekiq process was
+OOM-killed) cleanup middleware wasn't executed and locks were not released.
+
+If your job takes more than 15 minutes to complete, you can tune concurrency
+lock TTL to fit your needs:
+
+``` ruby
+# Set concurrency strategy lock TTL to 1 hour.
+sidekiq_throttle(:concurrency => { :limit => 20, :ttl => 1.hour.to_i })
+```
+
+
 ## Enhanced Queues list
 
 This gem provides ability to pause/resume queues from processing by workers.
@@ -155,9 +174,9 @@ end
 This library aims to support and is [tested against][travis] the following Ruby
 versions:
 
-* Ruby 2.3.x
 * Ruby 2.4.x
 * Ruby 2.5.x
+* Ruby 2.6.x
 
 If something doesn't work on one of these versions, it's a bug.
 

data/lib/sidekiq/throttled.rb

@@ -6,9 +6,11 @@ require "sidekiq"
 # internal
 require "sidekiq/throttled/version"
 require "sidekiq/throttled/communicator"
+require "sidekiq/throttled/configuration"
 require "sidekiq/throttled/queues_pauser"
 require "sidekiq/throttled/registry"
 require "sidekiq/throttled/worker"
+require "sidekiq/throttled/utils"
 
 # @see https://github.com/mperham/sidekiq/
 module Sidekiq
@@ -44,6 +46,13 @@ module Sidekiq
     private_constant :MUTEX
 
     class << self
+      include Utils
+
+      # @return [Configuration]
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
       # Hooks throttler into sidekiq.
       #
       # @return [void]
@@ -99,16 +108,6 @@ module Sidekiq
           @preloaded[job] ||= constantize(job) || true
         end
       end
-
-      # Resolve constant from it's name
-      def constantize(str)
-        str.sub(/^::/, "").split("::").inject(Object) do |const, name|
-          const.const_get(name)
-        end
-      rescue
-        Sidekiq.logger.warn { "Failed to constantize: #{str}" }
-        nil
-      end
     end
   end
 end

data/lib/sidekiq/throttled/configuration.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Throttled
+    # Configuration holder.
+    class Configuration
+      # Class constructor.
+      def initialize
+        reset!
+      end
+
+      # Reset configuration to defaults.
+      #
+      # @return [self]
+      def reset!
+        @inherit_strategies = false
+
+        self
+      end
+
+      # Instructs throttler to lookup strategies in parent classes, if there's
+      # no own strategy:
+      #
+      #     class Foo
+      #       include Sidekiq::Worker
+      #       include Sidekiq::Worker::Throttled
+      #
+      #       sidekiq_throttle :concurrency => { :limit => 42 }
+      #     end
+      #
+      #     class Bar < Foo
+      #     end
+      #
+      # By default in the example above, `Bar` won't have throttling options.
+      # Set this flag to `true` to enable this lookup in initializer, after
+      # that `Bar` will use `Foo` throttling bucket.
+      def inherit_strategies=(value)
+        @inherit_strategies = value ? true : false
+      end
+
+      # Whenever throttled workers should inherit parent's strategies or not.
+      # Default: `false`.
+      #
+      # @return [Boolean]
+      def inherit_strategies?
+        @inherit_strategies
+      end
+    end
+  end
+end

data/lib/sidekiq/throttled/queue_name.rb

@@ -27,7 +27,7 @@ module Sidekiq
         # @param [#to_s]
         # @return [String]
         def normalize(queue)
-          queue.to_s.sub(QUEUE_NAME_PREFIX_RE, "")
+          -queue.to_s.sub(QUEUE_NAME_PREFIX_RE, "")
         end
 
         # Prepends `queue:` prefix to given `queue` name.
@@ -38,7 +38,7 @@ module Sidekiq
         # @param [#to_s] queue Queue name
         # @return [String]
         def expand(queue)
-          "queue:#{queue}"
+          -"queue:#{queue}"
         end
       end
     end

data/lib/sidekiq/throttled/queues_pauser.rb

@@ -65,7 +65,10 @@ module Sidekiq
       # @private
       # @return [Array<String>]
       def filter(queues)
-        queues - @paused_queues.to_a
+        @mutex.synchronize { queues - @paused_queues.to_a }
+      rescue => e
+        Sidekiq.logger.error { "[#{self.class}] Failed filter queues: #{e}" }
+        queues
       end
 
       # Returns list of paused queues.

data/lib/sidekiq/throttled/registry.rb

@@ -2,6 +2,7 @@
 
 # internal
 require "sidekiq/throttled/strategy"
+require "sidekiq/throttled/utils"
 
 module Sidekiq
   module Throttled
@@ -13,6 +14,8 @@ module Sidekiq
       @aliases    = {}
 
       class << self
+        include Utils
+
         # Adds strategy to the registry.
         #
         # @note prints a warning to STDERR upon duplicate strategy name
@@ -49,12 +52,15 @@ module Sidekiq
         #
         # @overload get(name, &block)
         #   Yields control to the block if requested strategy was found.
+        #   @param [#to_s] name
         #   @yieldparam [Strategy] strategy
         #   @yield [strategy] Gives found strategy to the block
         #   @return result of a block
         def get(name)
-          strategy = @strategies[name.to_s] || @aliases[name.to_s]
+          strategy = find(name.to_s) || find_by_class(name)
+
           return yield strategy if strategy && block_given?
+
           strategy
         end
 
@@ -86,6 +92,34 @@ module Sidekiq
             yield(name, strategy) unless strategy.dynamic?
           end
         end
+
+        private
+
+        # Find strategy by it's name.
+        #
+        # @param name [String]
+        # @return [Strategy, nil]
+        def find(name)
+          @strategies[name] || @aliases[name]
+        end
+
+        # Find strategy by class or it's parents.
+        #
+        # @param name [Class, #to_s]
+        # @return [Strategy, nil]
+        def find_by_class(name)
+          return unless Throttled.configuration.inherit_strategies?
+
+          const = name.is_a?(Class) ? name : constantize(name)
+          return unless const.is_a?(Class)
+
+          const.ancestors.each do |m|
+            strategy = find(m.name)
+            return strategy if strategy
+          end
+
+          nil
+        end
       end
     end
   end

data/lib/sidekiq/throttled/strategy/concurrency.rb

@@ -42,7 +42,9 @@ module Sidekiq
 
         # @return [Boolean] whenever job is throttled or not
         def throttled?(jid, *job_args)
-          return false unless (job_limit = limit(job_args))
+          job_limit = limit(job_args)
+          return false unless job_limit
+          return true if job_limit <= 0
 
           kwargs = {
             :keys => [key(job_args)],

data/lib/sidekiq/throttled/strategy/threshold.rb

@@ -58,7 +58,9 @@ module Sidekiq
 
         # @return [Boolean] whenever job is throttled or not
         def throttled?(*job_args)
-          return false unless (job_limit = limit(job_args))
+          job_limit = limit(job_args)
+          return false unless job_limit
+          return true if job_limit <= 0
 
           kwargs = {
             :keys => [key(job_args)],

data/lib/sidekiq/throttled/utils.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Throttled
+    module Utils
+      module_function
+
+      # Resolve constant from it's name
+      # @param name [#to_s] Constant name
+      # @return [Object, nil] Resolved constant or nil if failed.
+      def constantize(name)
+        name.to_s.sub(/^::/, "").split("::").inject(Object, &:const_get)
+      rescue NameError
+        Sidekiq.logger.warn { "Failed to constantize: #{name}" }
+        nil
+      end
+    end
+  end
+end

data/lib/sidekiq/throttled/version.rb

@@ -3,6 +3,6 @@
 module Sidekiq
   module Throttled
     # Gem version
-    VERSION = "0.10.0.beta"
+    VERSION = "0.10.0"
   end
 end

data/sidekiq-throttled.gemspec

@@ -28,5 +28,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "redis-prescription"
   spec.add_runtime_dependency "sidekiq"
 
-  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "bundler", "~> 2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-throttled
 version: !ruby/object:Gem::Version
-  version: 0.10.0.beta
+  version: 0.10.0
 platform: ruby
 authors:
 - Alexey V Zapparov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-27 00:00:00.000000000 Z
+date: 2019-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.0'
 description: Concurrency and threshold throttling for Sidekiq.
 email:
 - ixti@member.fsf.org
@@ -94,6 +94,7 @@ files:
 - lib/sidekiq/throttled/communicator.rb
 - lib/sidekiq/throttled/communicator/callbacks.rb
 - lib/sidekiq/throttled/communicator/listener.rb
+- lib/sidekiq/throttled/configuration.rb
 - lib/sidekiq/throttled/errors.rb
 - lib/sidekiq/throttled/expirable_list.rb
 - lib/sidekiq/throttled/fetch.rb
@@ -110,6 +111,7 @@ files:
 - lib/sidekiq/throttled/strategy/threshold.lua
 - lib/sidekiq/throttled/strategy/threshold.rb
 - lib/sidekiq/throttled/testing.rb
+- lib/sidekiq/throttled/utils.rb
 - lib/sidekiq/throttled/version.rb
 - lib/sidekiq/throttled/web.rb
 - lib/sidekiq/throttled/web/queues.html.erb
@@ -134,12 +136,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Concurrency and threshold throttling for Sidekiq.