sidekiq-limit_fetch 2.1.1 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 37b25405100cca644241eb4c1106e31020ceaa71
-  data.tar.gz: c6e906b7f2d5553807fcf36082116f5fe7c06824
+  metadata.gz: e0a7907ad1ad1e09f8c873c8131ae4f3fa7625b6
+  data.tar.gz: 24fa5bd726c2a6ca724cc55775b361bb98a5b818
 SHA512:
-  metadata.gz: 0a3eeb2ee0098c67ae2971f4b62c4ab56526ddc1cd943d5cf39869d149baab4014382a6c696959a540d565584801e827facad77aad5286dbc8853aba872808ce
-  data.tar.gz: e7f166cb5cbadf788662befcca218b4a501343e6d05f2c9ff76199a293b301ee92137a822cbc66fb008e63c754e2d0ebdfc8457b80c497463748594244582d8d
+  metadata.gz: 7c7f3d96c4e0eafb5c22db40913f6e52b8ca674688d3f7d2a11c20318a06c4752c485c7268e7d643b18817a024e61b2bf3da2ccb8d84833c9a09b3042b06b937
+  data.tar.gz: 822d2ec33178bbcbbb620c8b699f2429a74c53ed5e66bb3eaa90811badb827f7862176361b59ecab1b584b63baede9e4f290080d9bc36f6227fe3dc29952c621

data/demo/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+gem 'rails'
+gem 'sinatra'
+gem 'celluloid'
+gem 'launchy'
+gem 'sidekiq-limit_fetch'
+

data/demo/README.md

@@ -0,0 +1,2 @@
+To test effect of limits run: `rake demo:limits`
+To change simulation see `lib/tasks/demo.rake`

data/demo/Rakefile

@@ -0,0 +1,61 @@
+require File.expand_path('../config/application', __FILE__)
+Demo::Application.load_tasks
+
+namespace :demo do
+  task limit: :environment do
+    puts '=> Creating sidekiq tasks'
+
+    100.times do
+      SlowWorker.perform_async
+      FastWorker.perform_async
+    end
+
+    run_sidekiq_monitoring
+    run_sidekiq_workers config: <<-YAML
+      :verbose: false
+      :concurrency: 4
+      :queues:
+        - slow
+        - fast
+      :limits:
+        slow: 1
+    YAML
+  end
+
+  def with_sidekiq_config(config)
+    whitespace_offset = config[/\A */].size
+    config.gsub! /^ {#{whitespace_offset}}/, ''
+
+    puts "=> Use sidekiq config:\n#{config}"
+    File.write 'config/sidekiq.yml', config
+    yield
+  ensure
+    FileUtils.rm 'config/sidekiq.yml'
+  end
+
+  def run_sidekiq_monitoring
+    require 'sidekiq/web'
+    Thread.new do
+      Rack::Server.start app: Sidekiq::Web, Port: 3000
+    end
+    sleep 1
+    Launchy.open 'http://127.0.0.1:3000/workers?poll=true'
+  end
+
+  def run_sidekiq_workers(options)
+    require 'sidekiq/cli'
+    cli = Sidekiq::CLI.instance
+
+    %w(validate! boot_system).each do |stub|
+      cli.define_singleton_method(stub) {}
+    end
+
+    with_sidekiq_config options[:config] do
+      config = cli.send :parse_config, 'config/sidekiq.yml'
+      Sidekiq.options.merge! config
+    end
+
+    cli.run
+  end
+end
+

data/demo/app/workers/fast_worker.rb

@@ -0,0 +1,8 @@
+class FastWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :fast
+
+  def perform
+    sleep 0.2
+  end
+end

data/demo/app/workers/slow_worker.rb

@@ -0,0 +1,8 @@
+class SlowWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :slow
+
+  def perform
+    sleep 1
+  end
+end

data/demo/config/application.rb

@@ -0,0 +1,11 @@
+require File.expand_path('../boot', __FILE__)
+
+require 'action_controller/railtie'
+require 'action_mailer/railtie'
+require 'sprockets/railtie'
+
+Bundler.require(:default, Rails.env)
+
+module Demo
+  Application = Class.new Rails::Application
+end

data/demo/config/boot.rb

@@ -0,0 +1,2 @@
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE'])

data/demo/config/environment.rb

@@ -0,0 +1,2 @@
+require File.expand_path('../application', __FILE__)
+Demo::Application.initialize!

data/demo/config/environments/development.rb

@@ -0,0 +1,9 @@
+Demo::Application.configure do
+  config.cache_classes = false
+  config.eager_load = false
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+  config.action_mailer.raise_delivery_errors = false
+  config.active_support.deprecation = :log
+  config.assets.debug = true
+end

data/lib/sidekiq/limit_fetch/global/monitor.rb

@@ -2,17 +2,16 @@ module Sidekiq::LimitFetch::Global
   module Monitor
     extend self
 
-    HEARTBEAT_NAMESPACE = 'heartbeat:'
-    PROCESSOR_NAMESPACE = 'processor:'
-
-    HEARTBEAT_TTL   = 18
+    HEARTBEAT_PREFIX = 'heartbeat:'
+    PROCESS_SET = 'processes'
+    HEARTBEAT_TTL = 18
     REFRESH_TIMEOUT = 10
 
     def start!(ttl=HEARTBEAT_TTL, timeout=REFRESH_TIMEOUT)
       Thread.new do
         loop do
           update_heartbeat ttl
-          invalidate_old_processors
+          invalidate_old_processes
           sleep timeout
         end
       end
@@ -23,35 +22,36 @@ module Sidekiq::LimitFetch::Global
     def update_heartbeat(ttl)
       Sidekiq.redis do |it|
         it.pipelined do
-          it.set processor_key, true
+          it.sadd PROCESS_SET, Selector.uuid
           it.set heartbeat_key, true
           it.expire heartbeat_key, ttl
         end
       end
     end
 
-    def invalidate_old_processors
+    def invalidate_old_processes
       Sidekiq.redis do |it|
-        it.keys(PROCESSOR_NAMESPACE + '*').each do |processor|
-          processor.sub! PROCESSOR_NAMESPACE, ''
-          next if it.get heartbeat_key processor
+        processes = it.smembers PROCESS_SET
+        processes.each do |process|
+          unless it.get heartbeat_key process
+            processes.delete process
+            it.srem PROCESS_SET, process
+          end
+        end
 
-          it.del processor_key processor
-          %w(limit_fetch:probed:* limit_fetch:busy:*).each do |pattern|
-            it.keys(pattern).each do |queue|
-              it.lrem queue, 0, processor
+        Sidekiq::Queue.instances.map(&:name).uniq.each do |queue|
+          locks = it.lrange "limit_fetch:probed:#{queue}", 0, -1
+          (locks.uniq - processes).each do |dead_process|
+            %w(limit_fetch:probed: limit_fetch:busy:).each do |prefix|
+              it.lrem prefix + queue, 0, dead_process
             end
           end
         end
       end
     end
 
-    def heartbeat_key(processor=Selector.uuid)
-      HEARTBEAT_NAMESPACE + processor
-    end
-
-    def processor_key(processor=Selector.uuid)
-      PROCESSOR_NAMESPACE + processor
+    def heartbeat_key(process=Selector.uuid)
+      HEARTBEAT_PREFIX + process
     end
   end
 end

data/sidekiq-limit_fetch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '2.1.1'
+  gem.version       = '2.1.2'
   gem.authors       = 'brainopia'
   gem.email         = 'brainopia@evilmartians.com'
   gem.summary       = 'Sidekiq strategy to support queue limits'

data/spec/sidekiq/limit_fetch/global/monitor_spec.rb

@@ -1,5 +1,7 @@
 require 'spec_helper'
 
+Thread.abort_on_exception = true
+
 describe Sidekiq::LimitFetch::Global::Monitor do
   let(:monitor) { described_class.start! ttl, timeout }
   let(:ttl) { 1 }
@@ -26,5 +28,15 @@ describe Sidekiq::LimitFetch::Global::Monitor do
       sleep 2*ttl
       queue.probed.should == 0
     end
+
+    it 'should remove invalid locks' do
+      2.times { queue.acquire }
+      described_class.stub :update_heartbeat
+      Sidekiq.redis do |it|
+        it.del Sidekiq::LimitFetch::Global::Monitor::PROCESS_SET
+      end
+      sleep 2*ttl
+      queue.probed.should == 0
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-limit_fetch
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.2
 platform: ruby
 authors:
 - brainopia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-22 00:00:00.000000000 Z
+date: 2013-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -73,6 +73,15 @@ files:
 - README.md
 - Rakefile
 - bench/compare.rb
+- demo/Gemfile
+- demo/README.md
+- demo/Rakefile
+- demo/app/workers/fast_worker.rb
+- demo/app/workers/slow_worker.rb
+- demo/config/application.rb
+- demo/config/boot.rb
+- demo/config/environment.rb
+- demo/config/environments/development.rb
 - lib/sidekiq-limit_fetch.rb
 - lib/sidekiq/extensions/queue.rb
 - lib/sidekiq/limit_fetch.rb
@@ -109,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Sidekiq strategy to support queue limits
@@ -120,3 +129,4 @@ test_files:
 - spec/sidekiq/limit_fetch/semaphore_spec.rb
 - spec/sidekiq/limit_fetch_spec.rb
 - spec/spec_helper.rb
+has_rdoc: