sidekiq-throttled 0.7.3 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84ceb88578f276b9161f1efa7428e46204da69e7
-  data.tar.gz: f57a36074d1d25b43125735f1a7fc8249d130533
+  metadata.gz: 3e18c53412711982bb7d620c5300d30d89cf1668
+  data.tar.gz: abe45ca2b495ee939a8ce11437b9e1fc48664242
 SHA512:
-  metadata.gz: f1f1e6086b2d493361a1413abecccdfaef578660f72fd6d46f41dc636fb143039cb54968eec7e497a12a43a70f6f1b74d78d573497f9d6141b1d390ee14677e7
-  data.tar.gz: b2bbc09d00d09c49ad5653d4eec477fb05049ac2423a3637cfdc5aae80d11e3808fd081fa9144757ff3f392ed45618f958e50ed0d4e264f844d6dd7960012f57
+  metadata.gz: e8d86da7c4b388b3b235caff8a887e342cae176d42398ef4033d946532cb6bd4568080db5b1afca15b6f6efe4984dd124229219c73524cb5bfe58ee813461fcb
+  data.tar.gz: 83e7c860e3b17b7ec8b4295ea6ddf5d6c9e8f9e984c4fa367d9d803cf2b7403e0bbafc0772df7b679e3e7ad0edfb47729646691eb913b10ff834c424aad97253

data/.rubocop.yml

@@ -6,6 +6,34 @@ AllCops:
   DisplayCopNames: true
   TargetRubyVersion: 2.4
 
+## Layout ######################################################################
+
+Layout/AlignHash:
+  EnforcedHashRocketStyle: table
+
+Layout/AlignParameters:
+  EnforcedStyle: with_fixed_indentation
+
+Layout/IndentArray:
+  EnforcedStyle: consistent
+
+Layout/IndentHash:
+  EnforcedStyle: consistent
+
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+
+Layout/SpaceInLambdaLiteral:
+  EnforcedStyle: require_space
+
+## Layout ######################################################################
+
+# Adds useless noise. I would prefer it to enforce blind rescues instead of
+# rescue from StandardError. To me blind rescue is as natural as implicit
+# return.
+Lint/RescueWithoutErrorClass:
+  Enabled: false
+
 ## Metrics #####################################################################
 
 Metrics/BlockLength:
@@ -15,12 +43,6 @@ Metrics/BlockLength:
 
 ## Styles ######################################################################
 
-Style/AlignHash:
-  EnforcedHashRocketStyle: table
-
-Style/AlignParameters:
-  EnforcedStyle: with_fixed_indentation
-
 Style/BracesAroundHashParameters:
   Enabled: false
 
@@ -33,21 +55,12 @@ Style/Encoding:
 Style/HashSyntax:
   EnforcedStyle: hash_rockets
 
-Style/IndentArray:
-  EnforcedStyle: consistent
-
-Style/IndentHash:
-  EnforcedStyle: consistent
-
 # Follow your heart where it makes sense to use lambda or lambda literal.
 # Enforcing it makes some pieces of code look REALLY terrible, e.g. in
 # case of empty (noop) lambdas: `lambda { |_| }`.
 Style/Lambda:
   Enabled: false
 
-Style/MultilineMethodCallIndentation:
-  EnforcedStyle: indented
-
 # Enabling this cop makes Guardfile (which is full of pathname regexps)
 # look absolutley style-inconsistent and terrible. In any case, this should
 # be on developer's choice whenever to use `%r` or not. Just like we don't
@@ -55,8 +68,9 @@ Style/MultilineMethodCallIndentation:
 Style/RegexpLiteral:
   Enabled: false
 
-Style/SpaceInLambdaLiteral:
-  EnforcedStyle: require_space
-
 Style/StringLiterals:
   EnforcedStyle: double_quotes
+
+# I prefer Yoda style instead, but there's no such enforcement style.
+Style/YodaCondition:
+  Enabled: false

data/Appraisals

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 appraise "sidekiq-4.0" do
   gem "sidekiq", "~> 4.0.0"
 end

data/CHANGES.md

@@ -1,3 +1,10 @@
+## 0.8.0 (2017-10-11)
+
+* Refactor concurrency throttling internals to use sorted sets in order to avoid
+  starvation in case when finalize! was not called (OOM / redis issues).
+  ([@ixti])
+
+
 ## 0.7.3 (2017-06-26)
 
 * [#34](https://github.com/sensortower/sidekiq-throttled/issues/34)

data/Gemfile

@@ -6,7 +6,7 @@ gem "appraisal"
 gem "rake"
 gem "redis-namespace", :require => false
 gem "rspec"
-gem "rubocop", "~> 0.47.0", :require => false
+gem "rubocop", "~> 0.50.0", :require => false
 gem "sidekiq"
 
 group :development do

data/Rakefile

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require "bundler/gem_tasks"
 
 require "rspec/core/rake_task"
@@ -20,7 +21,7 @@ namespace :rubocop do
   end
 end
 
-default_suite = ENV["CI"] ? :spec : %i(spec rubocop)
+default_suite = ENV["CI"] ? :spec : %i[spec rubocop]
 named_suites  = { "rubocop" => :rubocop, "rspec" => :spec }
 
 task :default => named_suites.fetch(ENV["SUITE"], default_suite)

data/lib/sidekiq/throttled.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 # 3rd party
 require "sidekiq"
 

data/lib/sidekiq/throttled/errors.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Sidekiq
   module Throttled
     # Generic class for Sidekiq::Throttled errors

data/lib/sidekiq/throttled/middleware.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 # internal
 require "sidekiq/throttled/registry"
 

data/lib/sidekiq/throttled/queues_pauser.rb

@@ -58,7 +58,7 @@ module Sidekiq
           end
 
           @communicator.ready do
-            @paused_queues.replace paused_queues.map { |q| QueueName.expand q }
+            @paused_queues.replace(paused_queues.map { |q| QueueName.expand q })
           end
         end
       end

data/lib/sidekiq/throttled/registry.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 # internal
 require "sidekiq/throttled/strategy"
 

data/lib/sidekiq/throttled/strategy.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 # internal
 require "sidekiq/throttled/errors"
 require "sidekiq/throttled/strategy/concurrency"

data/lib/sidekiq/throttled/strategy/concurrency.lua

@@ -2,12 +2,15 @@ local key = KEYS[1]
 local jid = ARGV[1]
 local lmt = tonumber(ARGV[2])
 local ttl = tonumber(ARGV[3])
+local now = tonumber(ARGV[4])
 
-if lmt <= redis.call("SCARD", key) and 0 == redis.call("SISMEMBER", key, jid) then
+redis.call("ZREMRANGEBYSCORE", key, "-inf", "(" .. now)
+
+if lmt <= redis.call("ZCARD", key) and not redis.call("ZSCORE", key, jid) then
   return 1
 end
 
-redis.call("SADD", key, jid)
+redis.call("ZADD", key, now + ttl, jid)
 redis.call("EXPIRE", key, ttl)
 
 return 0

data/lib/sidekiq/throttled/strategy/concurrency.rb

@@ -28,7 +28,7 @@ module Sidekiq
         # @param [#to_i] ttl Concurrency lock TTL in seconds.
         # @param [Proc] key_suffix Dynamic key suffix generator.
         def initialize(strategy_key, limit:, ttl: 900, key_suffix: nil)
-          @base_key   = "#{strategy_key}:concurrency"
+          @base_key   = "#{strategy_key}:concurrency.v2"
           @limit      = limit
           @ttl        = ttl.to_i
           @key_suffix = key_suffix
@@ -44,14 +44,14 @@ module Sidekiq
           return false unless (job_limit = limit(job_args))
 
           keys = [key(job_args)]
-          args = [jid.to_s, job_limit, @ttl]
+          args = [jid.to_s, job_limit, @ttl, Time.now.to_f]
 
           1 == SCRIPT.eval(keys, args)
         end
 
         # @return [Integer] Current count of jobs
         def count(*job_args)
-          Sidekiq.redis { |conn| conn.scard(key(job_args)) }.to_i
+          Sidekiq.redis { |conn| conn.zcard(key(job_args)) }.to_i
         end
 
         # Resets count of jobs
@@ -63,7 +63,7 @@ module Sidekiq
         # Remove jid from the pool of jobs in progress
         # @return [void]
         def finalize!(jid, *job_args)
-          Sidekiq.redis { |conn| conn.srem(key(job_args), jid.to_s) }
+          Sidekiq.redis { |conn| conn.zrem(key(job_args), jid.to_s) }
         end
       end
     end

data/lib/sidekiq/throttled/strategy/script.rb

@@ -41,7 +41,7 @@ module Sidekiq
         # Loads script to redis
         # @return [void]
         def bootstrap!
-          Sidekiq.redis do |conn|
+          namespaceless_redis do |conn|
             digest = conn.script(LOAD, @source)
 
             # XXX: this may happen **ONLY** if script digesting will be
@@ -76,6 +76,19 @@ module Sidekiq
         def self.read(file)
           new File.read file
         end
+
+        private
+
+        # Yields real namespace-less redis client.
+        def namespaceless_redis
+          Sidekiq.redis do |conn|
+            if defined?(Redis::Namespace) && conn.is_a?(Redis::Namespace)
+              conn = conn.redis
+            end
+
+            yield conn
+          end
+        end
       end
     end
   end

data/lib/sidekiq/throttled/testing.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require "sidekiq/throttled/registry"
 
 RSpec.configure do |config|

data/lib/sidekiq/throttled/version.rb

@@ -3,6 +3,6 @@
 module Sidekiq
   module Throttled
     # Gem version
-    VERSION = "0.7.3"
+    VERSION = "0.8.0"
   end
 end

data/lib/sidekiq/throttled/web/stats.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Sidekiq
   module Throttled
     module Web

data/lib/sidekiq/throttled/worker.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 # internal
 require "sidekiq/throttled/registry"
 

data/sidekiq-throttled.gemspec

@@ -1,4 +1,3 @@
-# coding: utf-8
 # frozen_string_literal: true
 
 lib = File.expand_path("../lib", __FILE__)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-throttled
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.8.0
 platform: ruby
 authors:
 - Alexey V Zapparov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-26 00:00:00.000000000 Z
+date: 2017-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -113,7 +113,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Concurrency and threshold throttling for Sidekiq.