our-eel-hacks 0.0.10 → 0.0.11

data/lib/our-eel-hacks/autoscaler.rb

@@ -68,6 +68,8 @@ module OurEelHacks
       @dynos = nil
       @soft_side = nil
 
+      @memoed_dyno_info = nil
+
       @last_scaled = Time.at(0)
       @entered_soft = Time.at(0)
       @last_reading = nil
@@ -80,8 +82,14 @@ module OurEelHacks
       @max_dynos = 10
       @lower_limits = LowerLimit.new(5, 1)
       @upper_limits = UpperLimit.new(30, 50)
-      @soft_duration = 500
-      @scaling_frequency = 200
+
+      @soft_duration = 10000
+      @scaling_frequency = 5000
+      @heroku_rate_limit = 80_000
+      @heroku_rate_limit_margin = 0.1
+
+      @millis_til_next_scale = nil
+
       @logger = NullLogger.new
     end
 
@@ -90,15 +98,20 @@ module OurEelHacks
       check_settings
       logger.info{ "Autoscaler configured for #{flavor || "{{unknown flavor}}"}"}
 
-      update_dynos(Time.now)
-      logger.debug{ self.inspect }
+      update_dynos(dyno_info.count, Time.now)
     end
 
+    MILLIS_PER_DAY = 24 * 60 * 60 * 1000
     def check_settings
       errors = []
       errors << "No heroku api key set" if @heroku_api_key.nil?
       errors << "No app name set" if @app_name.nil?
       errors << "No process type set" if @ps_type.nil?
+      if (MILLIS_PER_DAY / @heroku_rate_limit) *
+        (1.0 - @heroku_rate_limit_margin) *
+        API_CALLS_PER_SCALE > @scaling_frequency
+        errors << "Scaling frequency will lock up Heroku"
+      end
       unless errors.empty?
         logger.warn{ "Problems configuring Autoscaler: #{errors.inspect}" }
         raise "OurEelHacks::Autoscaler, configuration problem: " + errors.join(", ")
@@ -107,7 +120,7 @@ module OurEelHacks
 
     attr_accessor :min_dynos, :max_dynos, :lower_limits, :upper_limits, :ps_type,
       :soft_duration, :scaling_frequency, :logger, :heroku_api_key, :app_name
-    attr_reader :last_scaled, :dynos, :entered_soft, :last_reading, :soft_side
+    attr_reader :last_scaled, :dynos, :entered_soft, :last_reading, :soft_side, :millis_til_next_scale
 
     def elapsed(start, finish)
       seconds = finish.to_i - start.to_i
@@ -117,22 +130,29 @@ module OurEelHacks
       return diff
     end
 
+    API_CALLS_PER_SCALE = 2
     def scale(metric)
       logger.debug{ "Scaling request for #{@ps_type}: metric is: #{metric}" }
       moment = Time.now
-      if elapsed(last_scaled, moment) < scaling_frequency
-        logger.debug{ "Not scaling: elapsed #{elapsed(last_scaled, moment)} less than configured #{scaling_frequency}" }
+      if elapsed(last_scaled, moment) < millis_til_next_scale
+        logger.debug{ "Not scaling: elapsed #{elapsed(last_scaled, moment)} less than computed #{millis_til_next_scale}" }
         return
       end
 
+      clear_dyno_info
+
+      starting_wait = millis_til_next_scale
+
+      update_dynos(dyno_info.count, moment)
+
       target_dynos = target_scale(metric, moment)
 
       target_dynos = [[target_dynos, max_dynos].min, min_dynos].max
       logger.debug{ "Target dynos at: #{min_dynos}/#{target_dynos}/#{max_dynos} (vs. current: #{@dynos})" }
 
-      set_dynos(target_dynos)
+      set_dynos(target_dynos, moment)
 
-      update_dynos(moment)
+      break_cadence(starting_wait)
     rescue => ex
       logger.warn{ "Problem scaling: #{ex.inspect}" }
     end
@@ -173,16 +193,15 @@ module OurEelHacks
       return 0
     end
 
-    def dyno_info
-      regexp = /^#{ps_type}[.].*/
-      return heroku.ps(app_name).find_all do |dyno|
-        dyno["process"] =~ regexp
+    def break_cadence(starting_wait)
+      if starting_wait > millis_til_next_scale
+        @millis_til_next_scale = rand((starting_wait..@millis_til_next_scale))
       end
     end
 
-    def update_dynos(moment)
-      new_value = dyno_info.count
+    def update_dynos(new_value, moment)
       if new_value != dynos
+        @millis_til_next_scale = scaling_frequency * new_value
         @last_scaled = moment
         @entered_soft = moment
       end
@@ -190,6 +209,20 @@ module OurEelHacks
       @last_reading = moment
     end
 
+    def clear_dyno_info
+      @memoed_dyno_info = nil
+    end
+
+    def dyno_info
+      return @memoed_dyno_info ||=
+        begin
+          regexp = /^#{ps_type}[.].*/
+          heroku.ps(app_name).find_all do |dyno|
+            dyno["process"] =~ regexp
+          end
+        end
+    end
+
     def dynos_stable?
       return dyno_info.all? do |dyno|
         dyno["state"] == "up"
@@ -204,7 +237,7 @@ module OurEelHacks
       end
     end
 
-    def set_dynos(count)
+    def set_dynos(count,moment)
       if count == dynos
         logger.debug{ "Not scaling: #{count} ?= #{dynos}" }
         return
@@ -216,7 +249,7 @@ module OurEelHacks
       end
       logger.info{ "Scaling from #{dynos} to #{count} dynos for #{ps_type}" }
       heroku.ps_scale(app_name, :type => ps_type, :qty => count)
-      @last_scaled = Time.now
+      update_dynos(count, moment)
     end
   end
 end

data/spec/autoscaler.rb

@@ -1,4 +1,3 @@
-require 'spec_helper'
 require 'our-eel-hacks/rack'
 require 'logger'
 
@@ -18,11 +17,11 @@ describe OurEelHacks::Autoscaler do
   end
 
   let :scaling_freq do
-    200
+    2500
   end
 
   let :soft_dur do
-    500
+    5000
   end
 
   let :ideal_value do
@@ -45,6 +44,14 @@ describe OurEelHacks::Autoscaler do
     -10
   end
 
+  let :dyno_count do
+    3
+  end
+
+  let :expected_scale_frequency do
+    scaling_freq * dyno_count
+  end
+
   let! :starting_time do
     Time.now
   end
@@ -83,7 +90,7 @@ describe OurEelHacks::Autoscaler do
   end
 
   it "should get a count of dynos at start" do
-    autoscaler.dynos.should == 3 #happens to be the number of web dynos right now
+    autoscaler.dynos.should == dyno_count #comes from the VCR cassette
   end
 
   before :each do
@@ -95,14 +102,14 @@ describe OurEelHacks::Autoscaler do
   describe "scaling frequency" do
 
     it "should not scale too soon" do
-      time_adjust(scaling_freq - 5)
+      time_adjust(expected_scale_frequency - 5)
 
       heroku.should_not_receive(:ps_scale)
       autoscaler.scale(hard_high)
     end
 
     it "should scale up if time has elapsed and hard limit exceeded" do
-      time_adjust(scaling_freq + 5)
+      time_adjust(expected_scale_frequency + 5)
 
       heroku.should_receive(:ps_scale).with(app_name, hash_including(:qty => 4))
       autoscaler.scale(hard_high)
@@ -111,7 +118,7 @@ describe OurEelHacks::Autoscaler do
 
   describe "hard limits" do
     before :each do
-      time_adjust(scaling_freq + 5)
+      time_adjust(expected_scale_frequency + 5)
     end
 
     it "should scale down if hard lower limit exceeded" do
@@ -122,13 +129,13 @@ describe OurEelHacks::Autoscaler do
 
   describe "soft upper limit" do
     before :each do
-      time_adjust(scaling_freq * 2)
+      time_adjust(expected_scale_frequency * 2)
       autoscaler.scale(soft_high)
     end
 
     describe "if soft_duration hasn't elapsed" do
       before :each do
-        time_adjust((scaling_freq * 2) + soft_dur - 5)
+        time_adjust((expected_scale_frequency * 2) + soft_dur - 5)
         heroku.should_not_receive(:ps_scale)
       end
 
@@ -143,7 +150,7 @@ describe OurEelHacks::Autoscaler do
 
     describe "if soft_duration has elapsed" do
       before :each do
-        time_adjust(scaling_freq * 2 + soft_dur + 5)
+        time_adjust(expected_scale_frequency * 2 + soft_dur + 5)
       end
 
       it "should scale up if above upper soft limit" do
@@ -160,13 +167,13 @@ describe OurEelHacks::Autoscaler do
 
   describe "soft lower limit" do
     before :each do
-      time_adjust(scaling_freq * 2)
+      time_adjust(expected_scale_frequency * 2)
       autoscaler.scale(soft_low)
     end
 
     describe "if soft_duration hasn't elapsed" do
       before :each do
-        time_adjust(scaling_freq * 2 + soft_dur - 5)
+        time_adjust(expected_scale_frequency * 2 + soft_dur - 5)
         heroku.should_not_receive(:ps_scale)
       end
 
@@ -181,7 +188,7 @@ describe OurEelHacks::Autoscaler do
 
     describe "if soft_duration has elapsed" do
       before :each do
-        time_adjust(scaling_freq * 2 + soft_dur + 5)
+        time_adjust(expected_scale_frequency * 2 + soft_dur + 5)
       end
 
       it "should not scale up even if above upper soft limit" do

data/spec/rack.rb

@@ -1,4 +1,3 @@
-require 'spec_helper'
 require 'our-eel-hacks/rack'
 
 describe OurEelHacks::Rack do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: our-eel-hacks
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-09 00:00:00.000000000 Z
+date: 2012-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: corundum
-  requirement: &80593790 !ruby/object:Gem::Requirement
+  requirement: &75630260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.0.1
   type: :development
   prerelease: false
-  version_requirements: *80593790
+  version_requirements: *75630260
 - !ruby/object:Gem::Dependency
   name: heroku
-  requirement: &80592420 !ruby/object:Gem::Requirement
+  requirement: &75643080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>'
@@ -34,7 +34,7 @@ dependencies:
         - 0
   type: :runtime
   prerelease: false
-  version_requirements: *80592420
+  version_requirements: *75643080
 description: ! "  Middleware for Rack and Sidekiq to scale heroku.\n\n  A heroku process
   knows everything it needs in order to scale itself.  A little configuration, and
   you're set.\n"
@@ -66,7 +66,7 @@ rdoc_options:
 - --main
 - doc/README
 - --title
-- our-eel-hacks-0.0.10 RDoc
+- our-eel-hacks-0.0.11 RDoc
 require_paths:
 - lib/
 required_ruby_version: !ruby/object:Gem::Requirement
@@ -77,7 +77,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -794712049
+      hash: 838375187
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: