resque-heroku-scaler 0.2.1

data/HISTORY.md

@@ -0,0 +1,6 @@
+## 0.2.1 (2011-11-27)
+
+* Separate scaler process from worker hooks to prevent race conditions during worker shutdown
+* Scale manager for Heroku Cedar platform
+* Scale manager for local workers using rush
+* Core test coverage

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) Aaron Dunnington
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+Resque Heroku Scaler
+====================
+
+Provides autoscaling for [Resque][rq] workers on [Heroku][hk]. Based on
+previous scaling work developed by [Daniel Huckstep][dh] and
+[Alexander Murmann][am].
+
+Autoscaling behavior is provided through a separate monitor process. The
+scaler monitor process polls for pending jobs against the specified Resque
+Redis backend at a configurable interval. The scaler process runs as a worker
+process on Heroku.
+
+##Setup
+
+Add the following environment variables to your Heroku environment:
+
+* HEROKU_APP
+* HEROKU_USERNAME
+* HEROKU_PASSWORD
+
+In your Procfile, configure the scaler as a worker process using:
+
+```
+scaler: bundle exec rake resque:scaler:run
+```
+
+To run the scaler process, use the following command. Note, the scaler process
+is intended to run as a single instance.
+
+```
+heroku scale scaler=1
+```
+
+In your development environment, the scaler process can run local worker
+processes using the rush library. To configure, use the following in
+an initializer.
+
+```ruby
+require 'resque/plugins/resque-heroku-scaler'
+
+if Rails.env.development?
+  ENV["RUSH_PATH"] ||= File.expand_path('/path/to/app', __FILE__)
+  Resque::Plugins::ResqueHerokuScaler.configure do |c|
+    c.scale_manager = :local
+  end
+end
+```
+
+[rq]: http://github.com/defunkt/resque
+[hk]: http://devcenter.heroku.com/articles/cedar
+[dh]: http://verboselogging.com/2010/07/30/auto-scale-your-resque-workers-on-heroku
+[am]: https://github.com/ajmurmann/resque-heroku-autoscaler

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rake/testtask'
+
+task :default => :test
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*_test.rb']
+  t.verbose = true
+end

data/lib/resque/plugins/resque-heroku-scaler.rb

@@ -0,0 +1,6 @@
+require 'resque/plugins/resque_heroku_scaler/version'
+require 'resque/plugins/resque_heroku_scaler/config'
+require 'resque/plugins/resque_heroku_scaler/manager'
+require 'resque/plugins/resque_heroku_scaler/worker'
+require 'resque/plugins/resque_heroku_scaler/resque'
+require 'resque/plugins/resque_heroku_scaler'

data/lib/resque/plugins/resque_heroku_scaler/config.rb

@@ -0,0 +1,71 @@
+module Resque
+  module Plugins
+    module ResqueHerokuScaler
+      module Config
+        extend self
+
+        attr_writer :scale_manager
+        attr_writer :scale_interval
+        attr_writer :poll_interval
+        attr_writer :scale_timeout
+        attr_reader :scale_with
+
+        def scale_manager
+          @scale_manager || :heroku
+        end
+
+        def scale_interval
+          @scale_interval || 60
+        end
+
+        def poll_interval
+          @poll_interval || 5
+        end
+
+        def scale_timeout
+          @scale_timeout || 90
+        end
+
+        def scale_for(pending)
+          return @scale_with.call(pending) if @scale_with
+          default_scale_with(pending)
+        end
+
+        def scale_with=(block)
+          @scale_with = block
+        end
+
+        def default_scale_with(pending)
+          return 0 if pending <= 0
+
+          [{
+              :workers => 1,
+              :jobs => 1
+            },
+            {
+              :workers => 2,
+              :jobs => 15
+            },
+            {
+              :workers => 3,
+              :jobs => 25
+            },
+            {
+              :workers => 4,
+              :jobs => 40
+            },
+            {
+              :workers => 5,
+              :jobs => 60
+            }
+          ].reverse_each do |required_scale|
+            if pending >= required_scale[:jobs]
+              return required_scale[:workers]
+            end
+          end
+          return 0
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/resque_heroku_scaler/manager/heroku.rb

@@ -0,0 +1,25 @@
+require 'heroku'
+
+module Resque
+  module Plugins
+    module ResqueHerokuScaler
+      module Manager
+
+        class Heroku
+          def initialize(options={})
+            @heroku = ::Heroku::Client.new(ENV['HEROKU_USERNAME'], ENV['HEROKU_PASSWORD'])
+          end
+
+          def workers
+            @heroku.ps(ENV['HEROKU_APP']).count { |p| p["process"] =~ /worker\.\d?/ }
+          end
+
+          def workers=(qty)
+            @heroku.ps_scale(ENV['HEROKU_APP'], :type => 'worker', :qty => qty)
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/resque/plugins/resque_heroku_scaler/manager/local.rb

@@ -0,0 +1,56 @@
+require 'rush'
+
+module Resque
+  module Plugins
+    module ResqueHerokuScaler
+      module Manager
+
+        class Local
+
+          def initialize(options={})
+            @path = options[:path] || ENV['RUSH_PATH']
+            @processes = []
+          end
+
+          def workers
+            @processes.length
+          end
+
+          def workers=(qty)
+            active = workers
+            return if qty == active
+            if qty > active
+              scale_up(qty-active)
+              return
+            end
+            scale_down(active-qty)
+          end
+
+          def scale_up(qty)
+            qty.times do
+              process = Rush::Box.new[@path].bash('rake resque:work', :background => true, :env => { :BUNDLE_GEMFILE => '' })
+              @processes.push(process) if process
+            end
+          end
+
+          def scale_down(qty)
+            i = 0
+            until i == qty or @processes.empty?
+              process = @processes.pop
+              kill(process)
+              i += 1
+            end
+          end
+
+          def kill(process)
+            process.children.each do |child|
+              kill(child)
+            end
+            process.kill
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/resque/plugins/resque_heroku_scaler/manager.rb

@@ -0,0 +1,27 @@
+module Resque
+  module Plugins
+    module ResqueHerokuScaler
+      module Manager
+        extend self
+
+        def instance
+          @@instance ||= init_manager
+        end
+
+        def init_manager
+          handler = Resque::Plugins::ResqueHerokuScaler::Config.scale_manager
+          return handler unless [Symbol, Array, String].include? handler.class
+
+          options = {}
+          handler, options = handler if handler.is_a?(Array)
+          require File.dirname(__FILE__) + "/manager/#{handler}"
+          const_get(handler.to_s.capitalize).new(options)
+        end
+
+        def method_missing(m, *args)
+          instance.send(m, *args)
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/resque_heroku_scaler/resque.rb

@@ -0,0 +1,14 @@
+module Resque
+
+  alias_method :original_info, :info
+  
+  def scaling
+    Worker.scaling
+  end
+
+  def info
+    info_with_scale = original_info
+    info_with_scale[:scaling] = scaling.size
+    return info_with_scale
+  end
+end

data/lib/resque/plugins/resque_heroku_scaler/tasks.rb

@@ -0,0 +1,11 @@
+namespace :resque do
+  namespace :scaler do  
+    task :setup
+
+    desc "Start Resque Heroku Scaler process"
+    task :run => :setup do
+      require 'resque/plugins/resque-heroku-scaler'
+      Resque::Plugins::ResqueHerokuScaler.run
+    end
+  end
+end

data/lib/resque/plugins/resque_heroku_scaler/version.rb

@@ -0,0 +1,7 @@
+module Resque
+  module Plugins
+    module ResqueHerokuScaler
+      Version = VERSION = "0.2.1"
+    end
+  end
+end

data/lib/resque/plugins/resque_heroku_scaler/worker.rb

@@ -0,0 +1,85 @@
+module Resque
+
+  class Worker
+    alias_method :original_unregister_worker, :unregister_worker
+    
+    def work(interval = 5.0, &block)
+      interval = Float(interval)
+      $0 = "resque: Starting"
+      startup
+
+      loop do
+        wait_for_scale if scaling?
+        break if shutdown?
+
+        if not paused? and job = reserve
+          log "got: #{job.inspect}"
+          job.worker = self
+          run_hook :before_fork, job
+          working_on job
+
+          if @child = fork
+            srand # Reseeding
+            procline "Forked #{@child} at #{Time.now.to_i}"
+            Process.wait(@child)
+          else
+            procline "Processing #{job.queue} since #{Time.now.to_i}"
+            perform(job, &block)
+            exit! unless @cant_fork
+          end
+
+          done_working
+          @child = nil
+        else
+          break if interval.zero?
+          log! "Sleeping for #{interval} seconds"
+          procline paused? ? "Paused" : "Waiting for #{@queues.join(',')}"
+          sleep interval
+        end
+      end
+
+    ensure
+      unregister_worker
+    end
+
+    def wait_for_scale
+      redis.set("scale:#{self}", true)
+      sleep 1 while scaling? and not shutdown?
+      redis.del("scale:#{self}")
+    end
+
+    def unregister_worker
+      redis.del("scale:#{self}")
+      original_unregister_worker
+    end
+
+    def scaling?
+      redis.exists(:scale)
+    end
+
+    def self.scaling
+      names = all
+      return [] unless names.any?
+
+      names.map! { |name| "scale:#{name}" }
+
+      reportedly_scaling = {}
+
+      begin
+        reportedly_scaling = redis.mapped_mget(*names).reject do |key, value|
+          value.nil? || value.empty?
+        end
+      rescue Redis::Distributed::CannotDistribute
+        names.each do |name|
+          value = redis.get name
+          reportedly_scaling[name] = value unless value.nil? || value.empty?
+        end
+      end
+
+      reportedly_scaling.keys.map do |key|
+        find key.sub("scale:", '')
+      end.compact
+    end
+  end
+
+end

data/lib/resque/plugins/resque_heroku_scaler.rb

@@ -0,0 +1,106 @@
+module Resque
+  module Plugins
+
+    module ResqueHerokuScaler
+      class << self
+
+        def run
+          startup
+          loop do
+            begin
+              scale
+            rescue Exception => e
+              log "Scale failed with #{e.class.name} #{e.message}"
+            end
+            wait_for_scale
+          end
+        end
+
+        def scale
+          required = scale_for(pending)
+          active = workers
+
+          return if required == active
+
+          log "Scale workers from #{active} to #{required}"
+
+          if required > active
+            scale_workers(required)
+            return
+          end
+
+          signal_workers
+          stop = timeout
+          wait_for_workers until ready_to_scale(active) or timeout?(stop)
+          scale_workers(required)
+
+        ensure
+          resume_workers
+        end
+
+        def wait_for_scale
+          sleep Resque::Plugins::ResqueHerokuScaler::Config.scale_interval
+        end
+
+        def wait_for_workers
+          sleep Resque::Plugins::ResqueHerokuScaler::Config.poll_interval
+        end
+
+        def scale_for(pending)
+          Resque::Plugins::ResqueHerokuScaler::Config.scale_for(pending)
+        end
+
+        def scale_workers(qty)
+          Resque::Plugins::ResqueHerokuScaler::Manager.workers = qty
+        end
+
+        def workers
+          Resque::Plugins::ResqueHerokuScaler::Manager.workers
+        end
+
+        def signal_workers
+          Resque.redis.set(:scale, true)
+        end
+
+        def resume_workers
+          Resque.redis.del(:scale)
+        end
+
+        def timeout?(stop)
+          Time.now >= stop
+        end
+
+        def timeout
+          Time.now + Resque::Plugins::ResqueHerokuScaler::Config.scale_timeout
+        end
+
+        def pending
+          Resque.info[:pending].to_i
+        end
+
+        def ready_to_scale(active)
+          Resque.info[:scaling] == active
+        end
+
+        def configure
+          yield Resque::Plugins::ResqueHerokuScaler::Config
+        end
+
+        def startup
+          STDOUT.sync = true
+          trap('TERM') do
+            log "Shutting down scaler"
+            exit
+          end
+          log "Starting scaler"
+          resume_workers
+        end
+
+        def log(message)
+          puts "*** #{message}"
+        end
+      end
+
+    end
+  end
+end

data/test/config_test.rb

@@ -0,0 +1,61 @@
+require 'test_helper'
+
+class ConfigTest < MiniTest::Unit::TestCase
+  def setup
+    @config = Resque::Plugins::ResqueHerokuScaler::Config
+  end
+
+  def test_scale_manager_default
+    assert_equal :heroku, @config.scale_manager
+  end
+
+  def test_scale_interval_default
+    assert_equal 60, @config.scale_interval
+  end
+  
+  def test_poll_interval_default
+    assert_equal 5, @config.poll_interval
+  end
+  
+  def test_scale_timeout_default
+    assert_equal 90, @config.scale_timeout
+  end
+
+  def test_scale_for_default
+    assert_equal 2, @config.scale_for(20)
+  end
+
+  def test_scale_with_default
+    assert_equal nil, @config.scale_with
+  end
+
+  def test_custom_scale_with
+    @config.scale_with = Proc.new { |pending| 99 }
+    assert_equal 99, @config.scale_for(2)
+    @config.scale_with = nil
+  end
+
+  def test_scale_manager
+    @config.scale_manager = :local
+    assert_equal :local, @config.scale_manager
+    @config.scale_manager = :heroku
+  end
+
+  def test_scale_interval
+    @config.scale_interval = 99
+    assert_equal 99, @config.scale_interval
+    @config.scale_interval = 60
+  end
+
+  def test_poll_interval
+    @config.poll_interval = 99
+    assert_equal 99, @config.poll_interval
+    @config.poll_interval = 5
+  end
+  
+  def test_scale_timeout
+    @config.scale_timeout = 99
+    assert_equal 99, @config.scale_timeout
+    @config.scale_timeout = 90
+  end
+end

data/test/resque_heroku_scaler_test.rb

@@ -0,0 +1,68 @@
+require 'test_helper'
+
+class ResqueHerokuScalerTest < MiniTest::Unit::TestCase
+  def setup
+    @scaler = Resque::Plugins::ResqueHerokuScaler
+    @config = Resque::Plugins::ResqueHerokuScaler::Config
+
+    @redis = mock('Mock Redis')
+    @redis.stubs(:set).with(:scale, true).returns(true)
+    @redis.stubs(:del).with(:scale).returns(true)
+    Resque.stubs(:redis).returns(@redis)
+
+    @manager = mock('Mock Manager')
+    Resque::Plugins::ResqueHerokuScaler::Manager.stubs(:instance).returns(@manager)
+  end
+
+  def test_no_scale_for_zero_jobs
+    Resque.stubs(:info).returns({ :pending => 0, :scaling => 0 })
+    @manager.expects(:workers).returns(0)
+    @manager.expects(:workers=).never
+    @scaler.scale()
+  end
+
+  def test_scale_up_for_pending_job
+    Resque.stubs(:info).returns({ :pending => 1, :scaling => 0 })
+    @manager.expects(:workers).returns(0)
+    @manager.expects(:workers=).with(1)
+    @scaler.scale()
+  end
+
+  def test_scale_down_timeout
+    @config.scale_timeout = 1
+    Resque.stubs(:info).returns({ :pending => 0, :scaling => 0 })
+    @manager.expects(:workers).returns(1)
+    @manager.expects(:workers=).with(0)
+    @scaler.scale()
+    @config.scale_timeout = 90
+  end
+
+  def test_scale_down_for_zero_jobs
+    Resque.stubs(:info).returns({ :pending => 0, :scaling => 1 })
+    @manager.expects(:workers).returns(1)
+    @manager.expects(:workers=).with(0)
+    @scaler.scale()
+  end
+
+  def test_configure
+    @scaler.configure do |c|
+      c.scale_manager = :local
+      c.scale_interval = 30
+      c.poll_interval = 10
+      c.scale_timeout = 20
+      c.scale_with = Proc.new { |pending| 99 }
+    end
+
+    assert_equal :local, @config.scale_manager
+    assert_equal 30, @config.scale_interval
+    assert_equal 10, @config.poll_interval
+    assert_equal 20, @config.scale_timeout
+    assert_equal 99, @config.scale_for(2)
+
+    @config.scale_manager = :heroku
+    @config.scale_interval = 60
+    @config.poll_interval = 5
+    @config.scale_timeout = 90
+    @config.scale_with = nil
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require 'minitest/autorun'
+require 'mocha'
+require 'resque'
+require 'resque/plugins/resque-heroku-scaler'

data/test/worker_test.rb

@@ -0,0 +1,26 @@
+require 'test_helper'
+
+class WorkerTest < MiniTest::Unit::TestCase
+  def test_wait_for_scale
+    Resque.redis.stubs(:exists).with(:scale).returns(true)
+    Resque.redis.expects(:set).with(regexp_matches(/^scale:(.)*$/), true)
+    Resque.redis.expects(:del).with(regexp_matches(/^scale:(.)*$/))
+
+    worker = Resque::Worker.new(['*'])    
+    worker.stubs(:shutdown?).returns(true)
+    worker.stubs(:register_worker).returns(true)
+    worker.stubs(:unregister_worker).returns(true)
+    worker.work(0)
+  end
+
+  def test_unregister_worker
+    Resque.redis.expects(:del).with(regexp_matches(/^scale:(.)*$/))
+    Resque.redis.stubs(:del).with(regexp_matches(/^worker:(.)*$/))
+    Resque.redis.stubs(:del).with(regexp_matches(/^stat:(.)*$/))
+
+    worker = Resque::Worker.new(['*'])    
+    worker.stubs(:shutdown?).returns(true)
+    worker.stubs(:register_worker).returns(true)
+    worker.work(0)
+  end
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification 
+name: resque-heroku-scaler
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.2.1
+platform: ruby
+authors: 
+- Aaron Dunnington
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-11-27 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: resque
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.19.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: heroku
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 2.14.0
+  type: :runtime
+  version_requirements: *id002
+description: "    This gem provides autoscaling behavior for Resque jobs on Heroku.\n"
+email: spirogh@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.md
+- Rakefile
+- LICENSE
+- HISTORY.md
+- lib/resque/plugins/resque-heroku-scaler.rb
+- lib/resque/plugins/resque_heroku_scaler/config.rb
+- lib/resque/plugins/resque_heroku_scaler/manager/heroku.rb
+- lib/resque/plugins/resque_heroku_scaler/manager/local.rb
+- lib/resque/plugins/resque_heroku_scaler/manager.rb
+- lib/resque/plugins/resque_heroku_scaler/resque.rb
+- lib/resque/plugins/resque_heroku_scaler/tasks.rb
+- lib/resque/plugins/resque_heroku_scaler/version.rb
+- lib/resque/plugins/resque_heroku_scaler/worker.rb
+- lib/resque/plugins/resque_heroku_scaler.rb
+- test/config_test.rb
+- test/resque_heroku_scaler_test.rb
+- test/test_helper.rb
+- test/worker_test.rb
+has_rdoc: true
+homepage: http://github.com/spiro/resque-heroku-scaler
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.1
+signing_key: 
+specification_version: 3
+summary: Resque plugin to autoscale Heroku workers
+test_files: []
+