sidekiq-throttled 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76a708f284e2c4b3c3a65477f8893420854aeca4
-  data.tar.gz: 8c5c83a1d64063af01170067819bf167fe710bfe
+  metadata.gz: 8d035a47cfda4397f044828ef34ce2adfba25f4a
+  data.tar.gz: d41409aa677dd84626738d1f3860d39acd951d72
 SHA512:
-  metadata.gz: 6e703df521b67bccb01d0e022dd83376ac6d9cd68d57e7603f36212b7e8969a0128e5119df0e6224c9da813308a743e8ffb1302f44142adfc82b43b163c82978
-  data.tar.gz: b0b142ad1f692d962c7ffa63027c309350d5c0258a05d5b9cb1adcc2eb1a491244d0d9dd1191ef1348ed2930f020a1f6a66bc902ff6ab4bd2814c0b98a30233e
+  metadata.gz: f313f44d9f897fffc7bb1c3e503fdac7f6884bd3733eab89ede741a208b071207d7c7e09854fa631a00a0fde7308a6ba1753f8953211c66a55b3d53bfdac6342
+  data.tar.gz: 01f919a66399826c5e67f05b44e86d5440e5e63b6565b5cada2980a12d24f27e296d8d10f76ec4cd1d575e927d84666afa22066eb69d7c49d76374b3cd59cae8

data/.rubocop.yml

@@ -18,6 +18,9 @@ Style/EmptyLineBetweenDefs:
 Style/Encoding:
   EnforcedStyle: when_needed
 
+Style/FrozenStringLiteralComment:
+  EnforcedStyle: always
+
 Style/HashSyntax:
   EnforcedStyle: hash_rockets
 
@@ -34,8 +37,7 @@ Style/MultilineOperationIndentation:
 # IMHO `%r{foo/bar}` looks way more cleaner than `/foo\/bar/`.
 # Enabling this cop also makes Guardfile (which is full of pathname regexps)
 # look absolutley (style) inconsistent and terrible. Thus it should be on
-# developer's choice whenever to user `%r` or not. Just like we don't enforce
-# to use `["foo"]` over `%w(foo)` and so on.
+# developer's choice whenever to user `%r` or not.
 Style/RegexpLiteral:
   Enabled: false
 
@@ -45,7 +47,7 @@ Style/RegexpLiteral:
 #     conn.hset    :k1, now
 #     conn.hincrby :k2, 123
 #   end
-Style/SingleSpaceBeforeFirstArg:
+Style/SpaceBeforeFirstArg:
   Enabled: false
 
 Style/StringLiterals:

data/.travis.yml

@@ -1,4 +1,25 @@
 language: ruby
 rvm:
-  - 2.1.6
-before_install: gem install bundler -v 1.10.6
+  - 2.0
+  - 2.1
+  - 2.2
+  - 2.3.0
+  - ruby-head
+  - jruby-9.0.0.0
+  - jruby-9.0.1.0
+  - jruby-head
+  - ruby-head
+  - rbx-2
+matrix:
+  allow_failures:
+    - rvm: jruby-9.0.0.0
+    - rvm: jruby-9.0.1.0
+    - rvm: jruby-head
+    - rvm: ruby-head
+    - rvm: rbx-2
+  fast_finish: true
+before_install:
+  - gem install bundler -v 1.10.6
+services:
+  - redis-server
+sudo: false

data/CHANGES.md

@@ -0,0 +1,14 @@
+## 0.2.0 (2016-02-29)
+
+* [#6](https://github.com/sensortower/sidekiq-throttled/pull/6):
+  Add dynamic key suffix functionality.
+  ([@fhwang])
+
+
+## 0.1.0 (2015-11-03)
+
+* Initial release.
+
+
+[@ixti]: https://github.com/ixti
+[@fhwang]: https://github.com/fhwang

data/Gemfile

@@ -1,4 +1,15 @@
+# frozen_string_literal: true
 source "https://rubygems.org"
 
+group :test do
+  gem "coveralls"
+  gem "rack-test"
+  gem "rspec"
+  gem "rubocop"
+  gem "simplecov", ">= 0.9"
+  gem "sinatra", "~> 1.4", ">= 1.4.6"
+  gem "timecop"
+end
+
 # Specify your gem's dependencies in sidekiq-throttled.gemspec
 gemspec

data/README.md

@@ -1,6 +1,11 @@
 # Sidekiq::Throttled
 
-Concurrency and threshold throttling for Sidekiq.
+[![Gem Version](https://badge.fury.io/rb/sidekiq-throttled.svg)](http://rubygems.org/gems/sidekiq-throttled)
+[![Build Status](https://travis-ci.org/sensortower/sidekiq-throttled.svg?branch=master)](https://travis-ci.org/sensortower/sidekiq-throttled)
+[![Code Climate](https://codeclimate.com/github/sensortower/sidekiq-throttled.svg?branch=master)](https://codeclimate.com/github/sensortower/sidekiq-throttled)
+[![Coverage Status](https://coveralls.io/repos/sensortower/sidekiq-throttled/badge.svg?branch=master&service=github)](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
+
+Concurrency and threshold throttling for [Sidekiq](https://github.com/mperham/sidekiq/).
 
 
 ## Installation
@@ -58,6 +63,42 @@ end
 ```
 
 
+### Dynamic throttling
+
+You can throttle jobs dynamically with `:key_suffix` option:
+
+``` ruby
+class MyWorker
+  include Sidekiq::Worker
+  include Sidekiq::Throttled::Worker
+
+  sidekiq_options :queue => :my_queue
+
+  sidekiq_throttle({
+    # Allow maximum 10 concurrent jobs per user at a time.
+    :concurrency => { :limit => 10, :key_suffix => -> (user_id) { user_id } }
+  })
+
+  def perform(user_id)
+    # ...
+  end
+end
+```
+
+
+## Supported Ruby Versions
+
+This library aims to support and is [tested against][travis] the following Ruby
+versions:
+
+* Ruby 2.0.0
+* Ruby 2.1.x
+* Ruby 2.2.x
+* Ruby 2.3.0
+
+[travis]: http://travis-ci.org/sensortower/sidekiq-throttled
+
+
 ## Contributing
 
 * Fork sidekiq-throttled on GitHub

data/Rakefile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require "bundler/gem_tasks"
 
 require "rspec/core/rake_task"

data/bin/setup

@@ -1,7 +1,6 @@
 #!/bin/bash
+
 set -euo pipefail
 IFS=$'\n\t'
 
 bundle install
-
-# Do any other automated setup that you need to do here

data/lib/sidekiq/throttled.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # 3rd party
 require "sidekiq"
 
@@ -60,7 +61,7 @@ module Sidekiq
         jid = message.fetch("jid".freeze) { return false }
 
         Registry.get job do |strategy|
-          return strategy.throttled? jid
+          return strategy.throttled?(jid, *message["args"])
         end
 
         false

data/lib/sidekiq/throttled/basic_fetch.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # stdlib
 require "thread"
 
@@ -39,11 +40,11 @@ module Sidekiq
       # Tries to pop pair of `queue` and job `message` out of sidekiq queue.
       # @return [Array<String, String>, nil]
       def brpop
-        if @strictly_ordered_queues
-          queues = @unique_queues.dup
-        else
-          queues = @queues.shuffle.uniq
-        end
+        queues = if @strictly_ordered_queues
+                   @unique_queues.dup
+                 else
+                   @queues.shuffle.uniq
+                 end
 
         @mutex.synchronize do
           next if @suspended.empty?

data/lib/sidekiq/throttled/errors.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
   module Throttled
     # Generic class for Sidekiq::Throttled errors

data/lib/sidekiq/throttled/middleware.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # internal
 require "sidekiq/throttled/registry"
 
@@ -11,7 +12,7 @@ module Sidekiq
         yield
       ensure
         Registry.get msg["class".freeze] do |strategy|
-          strategy.finalize! msg["jid".freeze]
+          strategy.finalize!(msg["jid".freeze], *msg["args"])
         end
       end
     end

data/lib/sidekiq/throttled/registry.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # internal
 require "sidekiq/throttled/strategy"
 
@@ -34,7 +35,7 @@ module Sidekiq
           old_name = old_name.to_s
 
           warn "Duplicate strategy name: #{new_name}" if @strategies[new_name]
-          fail "Strategy not found: #{old_name}" unless @strategies[old_name]
+          raise "Strategy not found: #{old_name}" unless @strategies[old_name]
 
           @aliases[new_name] = @strategies[old_name]
         end
@@ -67,6 +68,21 @@ module Sidekiq
           @strategies.each { |*args| yield(*args) }
           self
         end
+
+        # @overload each_with_static_keys()
+        #   @return [Enumerator]
+        #
+        # @overload each_with_static_keys(&block)
+        #   @yieldparam [String] name
+        #   @yieldparam [Strategy] strategy
+        #   @yield [strategy] Gives strategy to the block
+        #   @return [Registry]
+        def each_with_static_keys
+          return to_enum(__method__) unless block_given?
+          @strategies.each do |name, strategy|
+            yield(name, strategy) unless strategy.dynamic_keys?
+          end
+        end
       end
     end
   end

data/lib/sidekiq/throttled/strategy.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # internal
 require "sidekiq/throttled/errors"
 require "sidekiq/throttled/strategy/concurrency"
@@ -15,28 +16,42 @@ module Sidekiq
       #   @return [Strategy::Threshold, nil]
       attr_reader :threshold
 
-      # @param [#to_s] key
+      # @param [#to_s] name
       # @param [Hash] concurrency Concurrency options.
       #   See {Strategy::Concurrency#initialize} for details.
       # @param [Hash] threshold Threshold options.
       #   See {Strategy::Threshold#initialize} for details.
-      def initialize(key, concurrency: nil, threshold: nil)
-        base_key      = "throttled:#{key}"
+      # @param [Hash] key_suffix Proc for dynamic keys.
+      def initialize(name, concurrency: nil, threshold: nil, key_suffix: nil)
+        key = "throttled:#{name}"
 
-        @concurrency  = concurrency && Concurrency.new(base_key, concurrency)
-        @threshold    = threshold && Threshold.new(base_key, threshold)
+        if concurrency
+          @concurrency = Concurrency.new(
+            key, concurrency.merge(:key_suffix => key_suffix)
+          )
+        end
+        if threshold
+          @threshold = Threshold.new(
+            key, threshold.merge(:key_suffix => key_suffix)
+          )
+        end
 
         return if @concurrency || @threshold
 
-        fail ArgumentError, "Neither :concurrency nor :threshold given"
+        raise ArgumentError, "Neither :concurrency nor :threshold given"
+      end
+
+      def dynamic_keys?
+        (@concurrency && @concurrency.dynamic_keys?) ||
+          (@threshold && @threshold.dynamic_keys?)
       end
 
       # @return [Boolean] whenever job is throttled or not.
-      def throttled?(jid)
-        return true if @concurrency && @concurrency.throttled?(jid)
+      def throttled?(jid, *job_args)
+        return true if @concurrency && @concurrency.throttled?(jid, *job_args)
 
-        if @threshold && @threshold.throttled?
-          finalize! jid
+        if @threshold && @threshold.throttled?(*job_args)
+          finalize!(jid, *job_args)
           return true
         end
 
@@ -45,8 +60,8 @@ module Sidekiq
 
       # Marks job as being processed.
       # @return [void]
-      def finalize!(jid)
-        @concurrency && @concurrency.finalize!(jid)
+      def finalize!(jid, *job_args)
+        @concurrency && @concurrency.finalize!(jid, *job_args)
       end
 
       # Resets count of jobs of all avaliable strategies

data/lib/sidekiq/throttled/strategy/concurrency.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # internal
 require "sidekiq/throttled/strategy/script"
 
@@ -20,38 +21,52 @@ module Sidekiq
         #   @return [Integer] Amount of allwoed concurrent job processors
         attr_reader :limit
 
-        # @param [#to_s] base_key
+        # @param [#to_s] strategy_key
         # @param [Hash] opts
         # @option opts [#to_i] :limit Amount of allwoed concurrent jobs
         #   processors running for given key
-        # @option opts [#to_i] :ttl (15.minutes) Concurrency lock TTL
-        def initialize(base_key, opts)
-          @key   = "#{base_key}:concurrency".freeze
-          @keys  = [@key]
+        # @option opts [#to_i] :ttl (15 minutes) Concurrency lock TTL
+        #   in seconds
+        # @option opts :key_suffix Proc for dynamic key suffix.
+        def initialize(strategy_key, opts)
+          @base_key = "#{strategy_key}:concurrency".freeze
           @limit = opts.fetch(:limit).to_i
-          @ttl   = opts.fetch(:ttl, 900).to_i
+          @ttl = opts.fetch(:ttl, 900).to_i
+          @key_suffix = opts[:key_suffix]
+        end
+
+        def dynamic_keys?
+          @key_suffix
         end
 
         # @return [Boolean] whenever job is throttled or not
-        def throttled?(jid)
-          1 == SCRIPT.eval(@keys, [@limit, @ttl, jid.to_s])
+        def throttled?(jid, *job_args)
+          1 == SCRIPT.eval([key(job_args)], [@limit, @ttl, jid.to_s])
         end
 
         # @return [Integer] Current count of jobs
-        def count
-          Sidekiq.redis { |conn| conn.scard(@key) }.to_i
+        def count(*job_args)
+          Sidekiq.redis { |conn| conn.scard(key(job_args)) }.to_i
         end
 
         # Resets count of jobs
         # @return [void]
-        def reset!
-          Sidekiq.redis { |conn| conn.del(@key) }.to_i
+        def reset!(*job_args)
+          Sidekiq.redis { |conn| conn.del(key(job_args)) }.to_i
         end
 
         # Remove jid from the pool of jobs in progress
         # @return [void]
-        def finalize!(jid)
-          Sidekiq.redis { |conn| conn.srem(@key, jid.to_s) }
+        def finalize!(jid, *job_args)
+          Sidekiq.redis { |conn| conn.srem(key(job_args), jid.to_s) }
+        end
+
+        private
+
+        def key(job_args)
+          key = @base_key.dup
+          key << ":#{@key_suffix.call(*job_args)}" if @key_suffix
+          key
         end
       end
     end

data/lib/sidekiq/throttled/strategy/script.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
   module Throttled
     class Strategy

data/lib/sidekiq/throttled/strategy/threshold.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # internal
 require "sidekiq/throttled/strategy/script"
 
@@ -36,31 +37,43 @@ module Sidekiq
         #   @return [Float] Period in seconds
         attr_reader :period
 
-        # @param [#to_s] base_key
+        # @param [#to_s] strategy_key
         # @param [Hash] opts
         # @option opts [#to_i] :limit Amount of jobs allowed per period
         # @option opts [#to_f] :period Period in seconds
-        def initialize(base_key, opts)
-          @key    = "#{base_key}:threshold".freeze
-          @keys   = [@key]
+        def initialize(strategy_key, opts)
+          @base_key = "#{strategy_key}:threshold".freeze
           @limit  = opts.fetch(:limit).to_i
           @period = opts.fetch(:period).to_f
+          @key_suffix = opts[:key_suffix]
+        end
+
+        def dynamic_keys?
+          @key_suffix
         end
 
         # @return [Boolean] whenever job is throttled or not
-        def throttled?
-          1 == SCRIPT.eval(@keys, [@limit, @period, Time.now.to_f])
+        def throttled?(*job_args)
+          1 == SCRIPT.eval([key(job_args)], [@limit, @period, Time.now.to_f])
         end
 
         # @return [Integer] Current count of jobs
-        def count
-          Sidekiq.redis { |conn| conn.llen(@key) }.to_i
+        def count(*job_args)
+          Sidekiq.redis { |conn| conn.llen(key(job_args)) }.to_i
         end
 
         # Resets count of jobs
         # @return [void]
-        def reset!
-          Sidekiq.redis { |conn| conn.del(@key) }
+        def reset!(*job_args)
+          Sidekiq.redis { |conn| conn.del(key(job_args)) }
+        end
+
+        private
+
+        def key(job_args)
+          key = @base_key.dup
+          key << ":#{@key_suffix.call(*job_args)}" if @key_suffix
+          key
         end
       end
     end

data/lib/sidekiq/throttled/testing.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require "sidekiq/throttled/registry"
 
 RSpec.configure do |config|

data/lib/sidekiq/throttled/version.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
 module Sidekiq
   module Throttled
     # Gem version
-    VERSION = "0.1.0".freeze
+    VERSION = "0.2.0".freeze
   end
 end

data/lib/sidekiq/throttled/web.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # stdlib
 require "pathname"
 

data/lib/sidekiq/throttled/web/index.html.erb

@@ -16,7 +16,7 @@
     </tr>
   </thead>
   <tbody>
-    <% Sidekiq::Throttled::Registry.each do |name, strategy| %>
+    <% Sidekiq::Throttled::Registry.each_with_static_keys do |name, strategy| %>
       <tr>
         <td style="vertical-align:middle;"><%= name %></td>
         <td style="vertical-align:middle;text-align:center;">

data/lib/sidekiq/throttled/web/stats.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
   module Throttled
     module Web
@@ -13,6 +14,9 @@ module Sidekiq
 
         # @param [Strategy::Concurrency, Strategy::Threshold] strategy
         def initialize(strategy)
+          if strategy && strategy.dynamic_keys?
+            raise ArgumentError, "Can't handle strategies with dynamic keys"
+          end
           @strategy = strategy
         end
 

data/lib/sidekiq/throttled/worker.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # internal
 require "sidekiq/throttled/registry"
 

data/sidekiq-throttled.gemspec

@@ -1,4 +1,5 @@
 # coding: utf-8
+# frozen_string_literal: true
 
 lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
@@ -16,8 +17,9 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/sensortower/sidekiq-throttled"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0")
-    .reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match %r{^(test|spec|features)/}
+  end
 
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -27,9 +29,4 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "timecop"
-  spec.add_development_dependency "rubocop"
-  spec.add_development_dependency "rack-test"
-  spec.add_development_dependency "sinatra", "~> 1.4", ">= 1.4.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-throttled
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Alexey V Zapparov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-03 00:00:00.000000000 Z
+date: 2016-02-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -52,82 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: timecop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rack-test
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: sinatra
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.4'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.4.6
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.4'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.4.6
 description: Concurrency and threshold throttling for Sidekiq.
 email:
 - ixti@member.fsf.org
@@ -140,11 +64,11 @@ files:
 - ".rubocop.yml"
 - ".travis.yml"
 - ".yardopts"
+- CHANGES.md
 - Gemfile
 - LICENSE.md
 - README.md
 - Rakefile
-- bin/console
 - bin/setup
 - lib/sidekiq/throttled.rb
 - lib/sidekiq/throttled/basic_fetch.rb
@@ -184,7 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.3
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Concurrency and threshold throttling for Sidekiq.

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "sidekiq/throttled"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start