sidekiq-limit_fetch 2.1.2 → 2.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e0a7907ad1ad1e09f8c873c8131ae4f3fa7625b6
-  data.tar.gz: 24fa5bd726c2a6ca724cc55775b361bb98a5b818
+  metadata.gz: d460d8048f5d1ef64e257f8af49ec1b7274eb966
+  data.tar.gz: 97b99fee5c035683ab9a8658d263190e0c5b2abd
 SHA512:
-  metadata.gz: 7c7f3d96c4e0eafb5c22db40913f6e52b8ca674688d3f7d2a11c20318a06c4752c485c7268e7d643b18817a024e61b2bf3da2ccb8d84833c9a09b3042b06b937
-  data.tar.gz: 822d2ec33178bbcbbb620c8b699f2429a74c53ed5e66bb3eaa90811badb827f7862176361b59ecab1b584b63baede9e4f290080d9bc36f6227fe3dc29952c621
+  metadata.gz: ebe51888c40335c3a8328536f5c3b57351bc856527c1b0acbb2f9e7818c70d911a3334735e123140b5aea04cce7571b564e8a4b74cc27725139337f1290c7169
+  data.tar.gz: b3e054be32248f8200615dc8c3b06e4f6d02c2e209eae5676fd8da44693b508f81c3ff3cf6add2a39a179fe18e963f8dda21136be5a6a9e59b846eefec8bcbb1

data/README.md

@@ -21,9 +21,11 @@ redis 2.6 or newer
 ```
 heroku uses redis 2.4 by default
 to update version you need to contact their support:
-https://devcenter.heroku.com/articles/redistogo#redis-26
+https://devcenter.heroku.com/articles/redistogo#redis-2-6
 ```
 
+**Important note:** At this moment, `sidekiq-limit_fetch` is incompatible with Sidekiq Pro's `reliable_fetch`.
+
 ## Usage
 
 ### Limits

data/demo/README.md

@@ -1,2 +1,2 @@
 To test effect of limits run: `rake demo:limits`
-To change simulation see `lib/tasks/demo.rake`
+To change simulation see `Rakefile`

data/lib/sidekiq/extensions/queue.rb

@@ -13,7 +13,8 @@ module Sidekiq
       :unblocked,     :block_except,
       :probed,        :busy,
       :increase_busy, :decrease_busy,
-      :local_busy?
+      :local_busy?,   :explain,
+      :remove_locks_except!
 
     def lock
       @lock ||= LimitFetch::Global::Semaphore.new name

data/lib/sidekiq/limit_fetch/global/monitor.rb

@@ -1,5 +1,6 @@
 module Sidekiq::LimitFetch::Global
   module Monitor
+    include Sidekiq::LimitFetch::Redis
     extend self
 
     HEARTBEAT_PREFIX = 'heartbeat:'
@@ -17,13 +18,29 @@ module Sidekiq::LimitFetch::Global
       end
     end
 
+    def all_processes
+      redis {|it| it.smembers PROCESS_SET }
+    end
+
+    def old_processes
+      all_processes.reject do |process|
+        redis {|it| it.get heartbeat_key process }
+      end
+    end
+
+    def remove_old_processes!
+      redis do |it|
+        old_processes.each {|process| it.srem PROCESS_SET, process }
+      end
+    end
+
     private
 
     def update_heartbeat(ttl)
       Sidekiq.redis do |it|
         it.pipelined do
-          it.sadd PROCESS_SET, Selector.uuid
           it.set heartbeat_key, true
+          it.sadd PROCESS_SET, Selector.uuid
           it.expire heartbeat_key, ttl
         end
       end
@@ -31,21 +48,11 @@ module Sidekiq::LimitFetch::Global
 
     def invalidate_old_processes
       Sidekiq.redis do |it|
-        processes = it.smembers PROCESS_SET
-        processes.each do |process|
-          unless it.get heartbeat_key process
-            processes.delete process
-            it.srem PROCESS_SET, process
-          end
-        end
+        remove_old_processes!
+        processes = all_processes
 
-        Sidekiq::Queue.instances.map(&:name).uniq.each do |queue|
-          locks = it.lrange "limit_fetch:probed:#{queue}", 0, -1
-          (locks.uniq - processes).each do |dead_process|
-            %w(limit_fetch:probed: limit_fetch:busy:).each do |prefix|
-              it.lrem prefix + queue, 0, dead_process
-            end
-          end
+        Sidekiq::Queue.instances.each do |queue|
+          queue.remove_locks_except! processes
         end
       end
     end

data/lib/sidekiq/limit_fetch/global/semaphore.rb

@@ -42,6 +42,10 @@ module Sidekiq::LimitFetch::Global
       redis {|it| it.llen "#{PREFIX}:busy:#@name" }
     end
 
+    def busy_processes
+      redis {|it| it.lrange "#{PREFIX}:busy:#@name", 0, -1 }
+    end
+
     def increase_busy
       increase_local_busy
       redis {|it| it.rpush "#{PREFIX}:busy:#@name", Selector.uuid }
@@ -56,6 +60,10 @@ module Sidekiq::LimitFetch::Global
       redis {|it| it.llen "#{PREFIX}:probed:#@name" }
     end
 
+    def probed_processes
+      redis {|it| it.lrange "#{PREFIX}:probed:#@name", 0, -1 }
+    end
+
     def pause
       redis {|it| it.set "#{PREFIX}:pause:#@name", true }
     end
@@ -96,5 +104,37 @@ module Sidekiq::LimitFetch::Global
     def local_busy?
       @local_busy > 0
     end
+
+    def explain
+      <<-END.gsub(/^ {8}/, '')
+        Current sidekiq process: #{Selector.uuid}
+
+          All processes:
+        #{Monitor.all_processes.join "\n"}
+
+          Stale processes:
+        #{Monitor.old_processes.join "\n"}
+
+          Locked queue processes:
+        #{probed_processes.sort.join "\n"}
+
+          Busy queue processes:
+        #{busy_processes.sort.join "\n"}
+      END
+    end
+
+    def remove_locks_except!(processes)
+      locked_processes = probed_processes.uniq
+      (locked_processes - processes).each do |dead_process|
+        remove_lock! dead_process
+      end
+    end
+
+    def remove_lock!(process)
+      redis do |it|
+        it.lrem "#{PREFIX}:probed:#@name", 0, process
+        it.lrem "#{PREFIX}:busy:#@name", 0, process
+      end
+    end
   end
 end

data/sidekiq-limit_fetch.gemspec

@@ -1,6 +1,7 @@
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '2.1.2'
+  gem.version       = '2.1.3'
+  gem.license       = 'MIT'
   gem.authors       = 'brainopia'
   gem.email         = 'brainopia@evilmartians.com'
   gem.summary       = 'Sidekiq strategy to support queue limits'

metadata

@@ -1,61 +1,61 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-limit_fetch
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.1.3
 platform: ruby
 authors:
 - brainopia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-04 00:00:00.000000000 Z
+date: 2014-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.6.5
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.6.5
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: |2
@@ -66,8 +66,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -100,7 +100,8 @@ files:
 - spec/sidekiq/limit_fetch_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/brainopia/sidekiq-limit_fetch
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -108,17 +109,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4
 summary: Sidekiq strategy to support queue limits
@@ -129,4 +130,3 @@ test_files:
 - spec/sidekiq/limit_fetch/semaphore_spec.rb
 - spec/sidekiq/limit_fetch_spec.rb
 - spec/spec_helper.rb
-has_rdoc: