resque-heroku-scaler 0.3.2 → 0.4.0

data/lib/resque/plugins/heroku_scaler.rb

@@ -29,30 +29,21 @@ module Resque
 
           return if required == active
 
-          log "Scale workers from #{active} to #{required}"
-
           if required > active
+            log "Scale workers from #{active} to #{required}"
             scale_workers(required)
             return
           end
+                      
+          return if pending?
 
-          signal_workers
-          stop = timeout
-          wait_for_workers until ready_to_scale(active) or timeout?(stop)
-          scale_workers(required)
-
-        ensure
-          resume_workers
+          scale_down(active)
         end
 
         def wait_for_scale
           sleep Resque::Plugins::HerokuScaler::Config.scale_interval
         end
 
-        def wait_for_workers
-          sleep Resque::Plugins::HerokuScaler::Config.poll_interval
-        end
-
         def scale_for(pending)
           Resque::Plugins::HerokuScaler::Config.scale_for(pending)
         end
@@ -61,32 +52,62 @@ module Resque
           Resque::Plugins::HerokuScaler::Manager.workers = qty
         end
 
+        def scale_down(active)
+          log "Scale #{active} workers down"
+
+          lock
+
+          timeout = Time.now + Resque::Plugins::HerokuScaler::Config.scale_timeout
+          until locked == active or Time.now >= timeout
+            sleep Resque::Plugins::HerokuScaler::Config.poll_interval
+          end
+
+          scale_workers(0)
+
+          timeout = Time.now + Resque::Plugins::HerokuScaler::Config.scale_timeout
+          until Time.now >= timeout
+            if offline?
+              log "#{active} workers scaled down successfully"
+              prune
+              break
+            end
+            sleep Resque::Plugins::HerokuScaler::Config.poll_interval
+          end
+
+        ensure
+          unlock
+        end
+
         def workers
           Resque::Plugins::HerokuScaler::Manager.workers
         end
-
-        def signal_workers
-          Resque.redis.set(:scale, true)
+        
+        def offline?
+          workers.zero?
         end
 
-        def resume_workers
-          Resque.redis.del(:scale)
+        def pending?
+          pending > 0
         end
 
-        def timeout?(stop)
-          Time.now >= stop
+        def pending
+          Resque.info[:pending]
         end
 
-        def timeout
-          Time.now + Resque::Plugins::HerokuScaler::Config.scale_timeout
+        def lock
+          Resque.lock
         end
 
-        def pending
-          Resque.info[:pending].to_i
+        def unlock
+          Resque.unlock
         end
-
-        def ready_to_scale(active)
-          Resque.info[:scaling] == active
+        
+        def locked
+          Resque.info[:locked]
+        end
+        
+        def prune
+          Resque.prune
         end
 
         def configure

data/lib/resque/plugins/heroku_scaler/config.rb

@@ -15,7 +15,7 @@ module Resque
         end
 
         def scale_interval
-          @scale_interval || 60
+          @scale_interval || 5
         end
 
         def poll_interval

data/lib/resque/plugins/heroku_scaler/resque.rb

@@ -1,14 +1,23 @@
 module Resque
 
   alias_method :original_info, :info
-  
-  def scaling
-    Worker.scaling
-  end
 
   def info
-    info_with_scale = original_info
-    info_with_scale[:scaling] = scaling.size
-    return info_with_scale
+    info_with_locked = original_info
+    info_with_locked[:locked] = Worker.locked.size
+    return info_with_locked
   end
+
+  def lock
+    Worker.lock
+  end
+
+  def unlock
+    Worker.unlock
+  end
+  
+  def prune
+    Worker.prune
+  end
+
 end

data/lib/resque/plugins/heroku_scaler/version.rb

@@ -1,7 +1,7 @@
 module Resque
   module Plugins
     module HerokuScaler
-      Version = VERSION = "0.3.2"
+      Version = VERSION = "0.4.0"
     end
   end
 end

data/lib/resque/plugins/heroku_scaler/worker.rb

@@ -1,18 +1,23 @@
 module Resque
-
+  
   class Worker
-    alias_method :original_unregister_worker, :unregister_worker
-    
+
     def work(interval = 5.0, &block)
       interval = Float(interval)
       $0 = "resque: Starting"
       startup
 
       loop do
-        wait_for_scale if scaling?
         break if shutdown?
+        
+        if should_lock?
+          lock
+          break
+        end
+
+        pause if should_pause?
 
-        if not paused? and job = reserve
+        if job = reserve(interval)
           log "got: #{job.inspect}"
           job.worker = self
           run_hook :before_fork, job
@@ -32,54 +37,57 @@ module Resque
           @child = nil
         else
           break if interval.zero?
-          log! "Sleeping for #{interval} seconds"
+          log! "Timed out after #{interval} seconds"
           procline paused? ? "Paused" : "Waiting for #{@queues.join(',')}"
-          sleep interval
         end
       end
 
     ensure
       unregister_worker
+      wait_for_shutdown if locked?
     end
 
-    def wait_for_scale
-      redis.set("scale:#{self}", true)
-      sleep 1 while scaling? and not shutdown?
-      redis.del("scale:#{self}")
+    def should_lock?
+      redis.exists(:lock)
     end
 
-    def unregister_worker
-      redis.del("scale:#{self}")
-      original_unregister_worker
+    def lock
+      redis.sadd(:locked, self)
+      @locked = true
     end
 
-    def scaling?
-      redis.exists(:scale)
+    def locked?
+      @locked
     end
 
-    def self.scaling
-      names = all
-      return [] unless names.any?
+    def should_unlock?
+      return false if should_lock?
+      locked?
+    end
 
-      names.map! { |name| "scale:#{name}" }
+    def wait_for_shutdown
+      sleep 0.1 until shutdown? or should_unlock?
+    end
 
-      reportedly_scaling = {}
+    def self.locked
+      Array(redis.smembers(:locked))
+    end
 
-      begin
-        reportedly_scaling = redis.mapped_mget(*names).reject do |key, value|
-          value.nil? || value.empty?
-        end
-      rescue Redis::Distributed::CannotDistribute
-        names.each do |name|
-          value = redis.get name
-          reportedly_scaling[name] = value unless value.nil? || value.empty?
-        end
-      end
+    def self.lock
+      redis.set(:lock, true)
+    end
 
-      reportedly_scaling.keys.map do |key|
-        find key.sub("scale:", '')
-      end.compact
+    def self.unlock
+      redis.del(:lock)
+      redis.del(:locked)
     end
-  end
 
-end
+    def self.prune
+      all_workers = Worker.all
+      all_workers.each do |worker|
+        worker.unregister_worker
+      end
+    end
+
+  end
+end

data/test/config_test.rb

@@ -10,7 +10,7 @@ class ConfigTest < MiniTest::Unit::TestCase
   end
 
   def test_scale_interval_default
-    assert_equal 60, @config.scale_interval
+    assert_equal 5, @config.scale_interval
   end
   
   def test_poll_interval_default
@@ -44,7 +44,7 @@ class ConfigTest < MiniTest::Unit::TestCase
   def test_scale_interval
     @config.scale_interval = 99
     assert_equal 99, @config.scale_interval
-    @config.scale_interval = 60
+    @config.scale_interval = 5
   end
 
   def test_poll_interval

data/test/heroku_scaler_test.rb

@@ -6,8 +6,8 @@ class HerokuScalerTest < MiniTest::Unit::TestCase
     @config = Resque::Plugins::HerokuScaler::Config
 
     @redis = mock('Mock Redis')
-    @redis.stubs(:set).with(:scale, true).returns(true)
-    @redis.stubs(:del).with(:scale).returns(true)
+    @redis.stubs(:set).with(:lock, true).returns(true)
+    @redis.stubs(:del).with(:lock).returns(true)
     Resque.stubs(:redis).returns(@redis)
 
     @manager = mock('Mock Manager')
@@ -15,14 +15,14 @@ class HerokuScalerTest < MiniTest::Unit::TestCase
   end
 
   def test_no_scale_for_zero_jobs
-    Resque.stubs(:info).returns({ :pending => 0, :scaling => 0 })
+    Resque.stubs(:info).returns({ :pending => 0, :locked => 0 })
     @manager.expects(:workers).returns(0)
     @manager.expects(:workers=).never
     @scaler.scale()
   end
 
   def test_scale_up_for_pending_job
-    Resque.stubs(:info).returns({ :pending => 1, :scaling => 0 })
+    Resque.stubs(:info).returns({ :pending => 1, :locked => 0 })
     @manager.expects(:workers).returns(0)
     @manager.expects(:workers=).with(1)
     @scaler.scale()
@@ -30,17 +30,25 @@ class HerokuScalerTest < MiniTest::Unit::TestCase
 
   def test_scale_down_timeout
     @config.scale_timeout = 1
-    Resque.stubs(:info).returns({ :pending => 0, :scaling => 0 })
+    Resque.stubs(:info).returns({ :pending => 0, :locked => 1 })
     @manager.expects(:workers).returns(1)
     @manager.expects(:workers=).with(0)
+    @scaler.expects(:offline?).returns(false)
+    @scaler.expects(:prune).never
+    @scaler.expects(:lock).returns(true)
+    @scaler.expects(:unlock).returns(true)      
     @scaler.scale()
     @config.scale_timeout = 90
   end
 
   def test_scale_down_for_zero_jobs
-    Resque.stubs(:info).returns({ :pending => 0, :scaling => 1 })
+    Resque.stubs(:info).returns({ :pending => 0, :locked => 1 })
     @manager.expects(:workers).returns(1)
     @manager.expects(:workers=).with(0)
+    @scaler.expects(:offline?).returns(true)
+    @scaler.expects(:prune).returns(true)
+    @scaler.expects(:lock).returns(true)
+    @scaler.expects(:unlock).returns(true)
     @scaler.scale()
   end
 

data/test/worker_test.rb

@@ -1,26 +1,15 @@
 require 'test_helper'
 
 class WorkerTest < MiniTest::Unit::TestCase
-  def test_wait_for_scale
-    Resque.redis.stubs(:exists).with(:scale).returns(true)
-    Resque.redis.expects(:set).with(regexp_matches(/^scale:(.)*$/), true)
-    Resque.redis.expects(:del).with(regexp_matches(/^scale:(.)*$/))
+  def test_lock
+    Resque.redis.stubs(:exists).with(:lock).returns(true)
+    Resque.redis.expects(:sadd).with(:locked, kind_of(Resque::Worker))
 
-    worker = Resque::Worker.new(['*'])    
-    worker.stubs(:shutdown?).returns(true)
+    worker = Resque::Worker.new(['*'])
     worker.stubs(:register_worker).returns(true)
     worker.stubs(:unregister_worker).returns(true)
-    worker.work(0)
-  end
-
-  def test_unregister_worker
-    Resque.redis.expects(:del).with(regexp_matches(/^scale:(.)*$/))
-    Resque.redis.stubs(:del).with(regexp_matches(/^worker:(.)*$/))
-    Resque.redis.stubs(:del).with(regexp_matches(/^stat:(.)*$/))
-
-    worker = Resque::Worker.new(['*'])    
-    worker.stubs(:shutdown?).returns(true)
-    worker.stubs(:register_worker).returns(true)
+    worker.stubs(:shutdown?).returns(false)
+    worker.expects(:wait_for_shutdown).returns(true)
     worker.work(0)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-heroku-scaler
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,30 +9,30 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-20 00:00:00.000000000Z
+date: 2012-06-28 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70164343744920 !ruby/object:Gem::Requirement
+  requirement: &70351775059460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.19.0
+        version: 1.20.0
   type: :runtime
   prerelease: false
-  version_requirements: *70164343744920
+  version_requirements: *70351775059460
 - !ruby/object:Gem::Dependency
   name: heroku
-  requirement: &70164343742680 !ruby/object:Gem::Requirement
+  requirement: &70351775058060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.18.1
+        version: 2.28.7
   type: :runtime
   prerelease: false
-  version_requirements: *70164343742680
+  version_requirements: *70351775058060
 description: ! '    This gem provides autoscaling for Resque workers on Heroku.
 
 '