resque-heroku-autoscaler 0.1.0 → 0.2.1

data/README.md

@@ -1,28 +1,59 @@
-Resque Heroku Autoscaler
-===========
+Resque Heroku Autoscaler (RHA)
+=======================
 
-A [Resque][rq] plugin. Requires Resque 1.8 and the Heroku gem (I only testet with 1.11.0).
+A [Resque][rq] plugin. Requires Resque 1.8 and the Heroku gem (only tested with 1.11.0).
 
 This gem scales your Heroku workers according to the number of pending Resque jobs. The original idea comes from Daniel Huckstep's [blog post on the topic][dh]
 
-Just extend your job class with esque::Plugins::HerokuAutoscaler.
+##Setup
 
+In order for the scaling to work RHA needs to know your **Heroku app's name, your Heroku user name and your Heroku password**. Per default those are being read from the following environment variables:
+
+- HEROKU_APP
+- HEROKU_USER
+- HEROKU_PASS
+
+If you prefer you also can configure RHA using a config block. For example you might put the following into config/initialiazers/resque_heroku_autoscaler_setup.config
+
+    require 'resque/plugins/resque_heroku_autoscaler'
+
+    Resque::Plugins::HerokuAutoscaler.config do |c|
+      c.heroku_user = 'john doe'
+      c.heroku_pass = ENV['HEROKU_PASSWORD']
+      c.heroku_app  = "my_app_#{Rails.env}"
+    end
+
+
+To use RHA in one of your jobs, just extend your job class with Resque::Plugins::HerokuAutoscaler.
 
 For example:
 
-    require 'resque/plugins/heroku_autoscaler'
+    require 'resque/plugins/resque_heroku_autoscaler'
 
     class TestJob
       extend Resque::Plugins::HerokuAutoscaler
 
       @queue = :test
 
-      def
+      def perform
+       ...awesome stuff...
+      end
     end
 
 When you add the job to your Resque queue, a new worker will be started if there isn't already one. If all jobs in the queue are processed the worker will be stopped again, keeping your costs low.
 
-Currently there is no way build in to set how many workers to are being started. I plan on adding that functionality soon.
+Per default RHA will only start a single worker, no matter how many jobs are pending. You can change this behavior in the config block as well:
+
+    require 'resque/plugins/resque_heroku_autoscaler'
+
+    Resque::Plugins::HerokuAutoscaler.config do |c|
+      c.new_worker_count do |pending|
+        (pending/5).ceil.to_i
+      end
+    end
+
+When calculating the new number of required workers the block given to new_worker_count will be called. Thus the example will result in starting one additional worker for every 5 pending jobs.
+
 
 [dh]: http://blog.darkhax.com/2010/07/30/auto-scale-your-resque-workers-on-heroku
 [rq]: http://github.com/defunkt/resque

data/lib/resque/plugins/heroku_autoscaler/config.rb

@@ -0,0 +1,34 @@
+module Resque
+  module Plugins
+    module HerokuAutoscaler
+      module Config
+        extend self
+
+        @new_worker_count = Proc.new {|pending| pending >0 ? 1 : 0}
+
+        attr_writer :heroku_user
+        def heroku_user
+          @heroku_user || ENV['HEROKU_USER']
+        end
+
+        attr_writer :heroku_pass
+        def heroku_pass
+          @heroku_pass || ENV['HEROKU_PASS']
+        end
+
+        attr_writer :heroku_app
+        def heroku_app
+          @heroku_app || ENV['HEROKU_APP']
+        end
+
+        def new_worker_count(pending=nil, *payload, &calculate_count)
+          if calculate_count
+            @new_worker_count = calculate_count
+          else
+            @new_worker_count.call(pending, *payload)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/heroku_autoscaler/version.rb

@@ -0,0 +1,7 @@
+module Resque
+  module Plugins
+    module HerokuAutoscaler
+      VERSION = "0.2.1"
+    end
+  end
+end

data/lib/resque/plugins/resque_heroku_autoscaler.rb

@@ -1,22 +1,35 @@
+require 'resque/plugins/heroku_autoscaler/config'
+
 module Resque
   module Plugins
     module HerokuAutoscaler
       @@heroku_client = nil
 
       def after_enqueue_scale_workers_up(*args)
-        set_workers(1)
+        set_workers(Resque::Plugins::HerokuAutoscaler::Config.new_worker_count(Resque.info[:pending]))
       end
 
       def after_perform_scale_workers_down(*args)
-        set_workers(0) if Resque.info[:pending] == 0
+        set_workers(Resque::Plugins::HerokuAutoscaler::Config.new_worker_count(Resque.info[:pending]))
       end
 
       def set_workers(number_of_workers)
-        heroku_client.set_workers(ENV['HEROKU_APP'], number_of_workers)
+        if number_of_workers != current_workers
+          heroku_client.set_workers(Resque::Plugins::HerokuAutoscaler::Config.heroku_app, number_of_workers)
+        end
+      end
+
+      def current_workers        
+        heroku_client.info(Resque::Plugins::HerokuAutoscaler::Config.heroku_app)[:workers].to_i
       end
 
       def heroku_client
-        @@heroku_client || @@heroku_client = Heroku::Client.new(ENV['HEROKU_USER'], ENV['HEROKU_PASS'])
+        @@heroku_client || @@heroku_client = Heroku::Client.new(Resque::Plugins::HerokuAutoscaler::Config.heroku_user,
+                                                                Resque::Plugins::HerokuAutoscaler::Config.heroku_pass)
+      end
+
+      def self.config
+        yield Resque::Plugins::HerokuAutoscaler::Config
       end
     end
   end

data/spec/config_spec.rb

@@ -0,0 +1,85 @@
+require 'rspec'
+require 'heroku'
+require 'resque'
+require 'resque/plugins/heroku_autoscaler/config'
+
+describe Resque::Plugins::HerokuAutoscaler::Config do
+  describe ".heroku_user" do
+    it "stores the given heroku user name" do
+      subject.heroku_user = "my_user@example.com"
+      subject.heroku_user.should == "my_user@example.com"
+    end
+
+    it "defaults to HEROKU_USER environment variable" do
+      subject.heroku_user = nil
+      ENV["HEROKU_USER"]  = "user@example.com"
+      subject.heroku_user.should == "user@example.com"
+    end
+  end
+
+  describe ".heroku_pass" do
+    it "stores the given heroku password" do
+      subject.heroku_pass = "password"
+      subject.heroku_pass.should == "password"
+    end
+
+    it "defaults to HEROKU_PASS environment variable" do
+      subject.heroku_pass = nil
+      ENV["HEROKU_PASS"]  = "123"
+      subject.heroku_pass.should == "123"
+    end
+  end
+
+  describe ".heroku_app" do
+    it "stores the given heroku application name" do
+      subject.heroku_app = "my-grand-app"
+      subject.heroku_app.should == "my-grand-app"
+    end
+
+    it "defaults to HEROKU_APP environment variable" do
+      subject.heroku_app = nil
+      ENV["HEROKU_APP"]  = "yaa"
+      subject.heroku_app.should == "yaa"
+    end
+  end
+
+  describe ".new_worker_count" do
+    before do
+      @original_method = Resque::Plugins::HerokuAutoscaler::Config.instance_variable_get(:@new_worker_count)
+    end
+
+    after do
+      Resque::Plugins::HerokuAutoscaler::Config.instance_variable_set(:@new_worker_count, @original_method)
+    end
+
+    it "should store a block as a Proc" do
+      subject.new_worker_count do |pending|
+        pending/5
+      end
+
+      subject.new_worker_count(10).should == 2
+    end
+
+    it "should be able to take the Resque job's payload as arguments" do
+      subject.new_worker_count do |pending, queue|
+        if queue == "test_queue"
+          10
+        else
+          pending/5
+        end
+      end
+
+      job_payload = ["test_queue", "more", "payload"]
+      subject.new_worker_count(10, *job_payload).should == 10
+    end
+
+    context "when the proc was not yet set" do
+      before do
+        subject.new_worker_count do |pending, queue|
+        end
+        it { subject.new_worker_count(0).should == 0 }
+        it { subject.new_worker_count(1).should == 1 }
+      end
+    end
+  end
+end

data/spec/resque_heroku_autoscaler_spec.rb

@@ -20,6 +20,12 @@ RSpec.configure do |config|
 end
 
 describe Resque::Plugins::HerokuAutoscaler do
+  before do
+    @fake_heroku_client = Object.new
+    stub(@fake_heroku_client).set_workers
+    stub(@fake_heroku_client).info { {:workers => 0} }
+  end
+
   it "should be a valid Resque plugin" do
     lambda { Resque::Plugin.lint(Resque::Plugins::HerokuAutoscaler) }.should_not raise_error
   end
@@ -30,25 +36,51 @@ describe Resque::Plugins::HerokuAutoscaler do
     end
 
     it "should take whatever args Resque hands in" do
-      stub(Heroku::Client).new { stub!.set_workers }
+      stub(TestJob).heroku_client { @fake_heroku_client }
 
-      lambda { TestJob.after_enqueue_scale_workers_up("some", "random", "aguments", 42) }.should_not raise_error
+      lambda do
+        TestJob.after_enqueue_scale_workers_up("some", "random", "aguments", 42) 
+      end.should_not raise_error
     end
 
     it "should create one worker" do
       stub(TestJob).workers { 0 }
+      stub(Resque).info{ {:pending => 1} }
       mock(TestJob).set_workers(1)
       TestJob.after_enqueue_scale_workers_up
     end
+
+    context "when new_worker_count was changed" do
+      before do
+        @original_method = Resque::Plugins::HerokuAutoscaler::Config.instance_variable_get(:@new_worker_count)
+        subject.config do |c|
+          c.new_worker_count do
+            2
+          end
+        end
+      end
+
+      after do
+        Resque::Plugins::HerokuAutoscaler::Config.instance_variable_set(:@new_worker_count, @original_method)
+      end
+
+      it "should use the given block" do
+        mock(TestJob).set_workers(2)
+        TestJob.after_enqueue_scale_workers_up
+      end
+    end
   end
 
   describe ".after_perform_scale_workers_down" do
+    before do
+      stub(TestJob).heroku_client { @fake_heroku_client }
+    end
 
     it "should add the hook" do
       Resque::Plugin.after_hooks(TestJob).should include("after_perform_scale_workers_down")
     end
 
-    it "should take whatever args Resque hands in" do      
+    it "should take whatever args Resque hands in" do
       Resque::Plugins::HerokuAutoscaler.class_eval("@@heroku_client = nil")
       stub(Heroku::Client).new { stub!.set_workers }
 
@@ -71,8 +103,28 @@ describe Resque::Plugins::HerokuAutoscaler do
         stub(Resque).info { {:pending => 1} }
       end
 
-      it "should not change workers" do
-        dont_allow(TestJob).set_workers
+      it "should keep workers at 1" do
+        mock(TestJob).set_workers(1)
+        TestJob.after_perform_scale_workers_down
+      end
+    end
+
+    context "when new_worker_count was changed" do
+      before do
+        @original_method = Resque::Plugins::HerokuAutoscaler::Config.instance_variable_get(:@new_worker_count)
+        subject.config do |c|
+          c.new_worker_count do
+            2
+          end
+        end
+      end
+
+      after do
+        Resque::Plugins::HerokuAutoscaler::Config.instance_variable_set(:@new_worker_count, @original_method)
+      end
+
+      it "should use the given block" do
+        mock(TestJob).set_workers(2)
         TestJob.after_perform_scale_workers_down
       end
     end
@@ -80,16 +132,22 @@ describe Resque::Plugins::HerokuAutoscaler do
 
   describe ".set_workers" do
     it "should use the Heroku client to set the workers" do
-      ENV['HEROKU_APP'] = 'some app name'
-      mock(TestJob).heroku_client { mock!.set_workers('some app name', 10) }
+      subject.config do |c|
+        c.heroku_app = 'some_app_name'
+      end
+
+      stub(TestJob).current_workers {0}
+      mock(TestJob).heroku_client { p "returning"; mock(@fake_heroku_client).set_workers('some_app_name', 10) }
       TestJob.set_workers(10)
     end
   end
 
   describe ".heroku_client" do
     before do
-      ENV['HEROKU_USER'] = 'john doe'
-      ENV['HEROKU_PASS'] = 'password'
+      subject.config do |c|
+        c.heroku_user = 'john doe'
+        c.heroku_pass = 'password'
+      end
     end
 
     it "should return a heroku client" do
@@ -116,4 +174,23 @@ describe Resque::Plugins::HerokuAutoscaler do
       TestJob.heroku_client.should == AnotherJob.heroku_client
     end
   end
-end
+
+  describe ".config" do
+    it "yields the configuration" do
+      subject.config do |c|
+        c.should == Resque::Plugins::HerokuAutoscaler::Config
+      end
+    end
+  end
+
+  describe ".current_workers" do
+    it "should request the numbers of active workers from Heroku" do
+      subject.config do |c|
+        c.heroku_app = "my_app"
+      end
+
+      mock(TestJob).heroku_client { mock!.info("my_app") { {:workers => 10} } }
+      TestJob.current_workers.should == 10
+    end
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: resque-heroku-autoscaler
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 21
   prerelease: 
   segments: 
   - 0
+  - 2
   - 1
-  - 0
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors: 
 - Alexander Murmann
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-06 00:00:00 -08:00
+date: 2011-02-11 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -47,7 +47,9 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id002
-description: 
+description: |
+  This gem scales your Heroku workers according to the number of pending Resque jobs. You can customize the scaling behavior of your workers, however you like.
+
 email: ajmurmann@gmail.com
 executables: []
 
@@ -59,7 +61,10 @@ files:
 - README.md
 - MIT.LICENSE
 - GEMFILE
+- lib/resque/plugins/heroku_autoscaler/config.rb
+- lib/resque/plugins/heroku_autoscaler/version.rb
 - lib/resque/plugins/resque_heroku_autoscaler.rb
+- spec/config_spec.rb
 - spec/resque_heroku_autoscaler_spec.rb
 has_rdoc: true
 homepage: https://github.com/ajmurmann/resque-heroku-autoscaler