our-eel-hacks 0.0.1

data/lib/our-eel-hacks/autoscaler.rb

@@ -0,0 +1,199 @@
+require 'heroku'
+
+module OurEelHacks
+  class Autoscaler
+    class << self
+      def get_instance(flavor)
+        flavor = flavor.to_sym
+        @instances ||= Hash.new{ |h,k| h[k] = self.new }
+        return @instances[flavor]
+      end
+
+      def configure(flavor = :web, &block)
+        get_instance(flavor).configure(&block)
+      end
+
+      def instance_for(flavor = :web)
+        instance = get_instance(flavor)
+        instance.check_settings
+        return instance
+      end
+    end
+
+    class Limit
+      def initialize(soft, hard)
+        @soft = soft
+        @hard = hard
+      end
+
+      attr_accessor :hard, :soft
+    end
+
+    class UpperLimit < Limit
+      def includes?(value)
+        return (@soft < value && value <= @hard)
+      end
+
+      def >(value)
+        return @soft > value
+      end
+
+      def <(value)
+        return @hard < value
+      end
+    end
+
+    class LowerLimit < Limit
+      def includes?(value)
+        return (value >= @hard && value <= @soft)
+      end
+
+      def >(value)
+        return @hard > value
+      end
+
+      def <(value)
+        return @soft < value
+      end
+    end
+
+    def initialize()
+      @dynos = nil
+      @soft_side = nil
+
+      @last_scaled = 0
+      @entered_soft = nil
+      @last_reading = nil
+
+      @app_name = nil
+      @ps_type = nil
+      @heroku_api_key = nil
+
+      @min_dynos = 1
+      @max_dynos = 10
+      @lower_limits = LowerLimit.new(5, 1)
+      @upper_limits = UpperLimit.new(30, 50)
+      @soft_duration = 500
+      @scaling_frequency = 200
+      @logger = nil
+    end
+
+    def log(msg)
+      return if @logger.nil
+      @logger.info(msg)
+    end
+
+    def configure
+      yield self
+      check_settings
+
+      update_dynos(Time.now)
+    end
+
+    def check_settings
+      errors = []
+      errors << "No heroku api key set" if @heroku_api_key.nil?
+      errors << "No app name set" if @app_name.nil?
+      errors << "No process type set" if @ps_type.nil?
+      raise "OurEelHacks::Autoscaler, configuration problem: " + errors.join(", ") unless errors.empty?
+    end
+
+    attr_accessor :min_dynos, :max_dynos, :lower_limits, :upper_limits, :ps_type,
+      :soft_duration, :scaling_frequency, :logger, :heroku_api_key, :app_name
+    attr_reader :last_scaled, :dynos, :entered_soft, :last_reading, :soft_side
+
+    def elapsed(start, finish)
+      seconds = finish.to_i - start.to_i
+      millis = finish.usec - start.usec
+      return seconds * 1000 + millis
+    end
+
+    def scale(metric)
+      moment = Time.now
+      if elapsed(last_scaled, moment) < scaling_frequency
+        return
+      end
+
+      target_dynos = target_scale(metric, moment)
+
+      target_dynos = [[target_dynos, max_dynos].min, min_dynos].max
+
+      set_dynos(target_dynos)
+
+      update_dynos(moment)
+    end
+
+    def target_scale(metric, moment)
+      if lower_limits > metric
+        return dynos - 1
+      elsif upper_limits < metric
+        return dynos + 1
+      elsif
+        result = (dynos + soft_limit(metric, moment))
+        return result
+      end
+    end
+
+    def soft_limit(metric, moment)
+      hit_limit = [lower_limits, upper_limits].find{|lim| lim.includes? metric}
+
+      if soft_side == hit_limit
+        if elapsed(entered_soft, moment) > soft_duration
+          entered_soft = moment
+          case hit_limit
+          when upper_limits
+            return +1
+          when lower_limits
+            return -1
+          else
+            return 0
+          end
+        else
+          return 0
+        end
+      else
+        @entered_soft = moment
+      end
+
+      @soft_side = hit_limit
+      return 0
+    end
+
+    def dyno_info
+      regexp = /^#{ps_type}[.].*/
+      return heroku.ps(app_name).find_all do |dyno|
+        dyno["process"] =~ regexp
+      end
+    end
+
+    def update_dynos(moment)
+      new_value = dyno_info.count
+      if new_value != dynos
+        @last_scaled = moment
+        @entered_soft = moment
+      end
+      @dynos = new_value
+      @last_reading = moment
+    end
+
+    def dynos_stable?
+      return dyno_info.all? do |dyno|
+        dyno["state"] == "up"
+      end
+    end
+
+    def heroku
+      @heroku ||= Heroku::Client.new("", heroku_api_key).tap do |client|
+        unless client.info(app_name)[:stack] == "cedar"
+          raise "#{self.class.name} built against cedar stack"
+        end
+      end
+    end
+
+    def set_dynos(count)
+      return if count == dynos or not dynos_stable?
+      heroko.ps_scale(app_name, :type => ps_type, :qty => count)
+      @last_scaled = Time.now
+    end
+  end
+end

data/lib/our-eel-hacks/defer/event-machine.rb

@@ -0,0 +1,12 @@
+require 'eventmachine'
+module OurEelHacks
+  module Defer
+    module EventMachine
+      def autoscale(*args)
+        EM.defer do
+          super
+        end
+      end
+    end
+  end
+end

data/lib/our-eel-hacks/middleware.rb

@@ -0,0 +1,15 @@
+require 'our-eel-hacks/autoscaler'
+
+module OurEelHacks
+  class Middleware
+    def initialize(flavor)
+      @flavor = flavor
+    end
+
+    protected
+
+    def autoscale(metric)
+      Autoscaler.instance_for(@flavor).scale(metric)
+    end
+  end
+end

data/lib/our-eel-hacks/rack.rb

@@ -0,0 +1,29 @@
+require 'our-eel-hacks/middleware'
+require 'our-eel-hacks/defer/event-machine'
+module OurEelHacks
+  class Rack < Middleware
+    include Defer::EventMachine
+
+    def initialize(app, env_field, flavor = :web)
+      super(flavor)
+      @env_field = env_field
+      @app = app
+    end
+
+    def call(env)
+      autoscale(metric_from(env))
+    ensure
+      @app.call(env)
+    end
+
+    def metric_from(env)
+      Integer(env[@env_field]) rescue 0
+    end
+  end
+
+  class ScaleOnRoutingQueue < Rack
+    def initialize(app, flavor = :web)
+      super(app, "HTTP_X_HEROKU_QUEUE_DEPTH", flavor)
+    end
+  end
+end

data/lib/our-eel-hacks/sidekiq.rb

@@ -0,0 +1,23 @@
+require 'our-eel-hacks/middleware'
+require 'our-eel-hacks/defer/event-machine'
+
+module OurEelHacks
+  class Sidekiq < Middleware
+    include Defer::EventMachine
+    def initialize(flavor=:sidekiq)
+      super
+    end
+
+    def call(worker_class, item, queue)
+      autoscale(get_queue_length(queue))
+    ensure
+      yield
+    end
+
+    def get_queue_length(queue)
+      ::Sidekiq.redis do |conn|
+        conn.llen("queue:#{queue}") || 0
+      end
+    end
+  end
+end

data/spec/autoscaler.rb

@@ -0,0 +1,187 @@
+require 'spec_helper'
+require 'our-eel-hacks/rack'
+
+describe OurEelHacks::Autoscaler do
+  before :each do
+    FakeWeb.allow_net_connect = false
+  end
+
+  use_vcr_cassette :record => :once
+
+  let :app_name do
+    "sbmp"
+  end
+
+  let :api_key do
+    "FakeApiKey"
+  end
+
+  let :scaling_freq do
+    200
+  end
+
+  let :soft_dur do
+    500
+  end
+
+  let :ideal_value do
+    25
+  end
+
+  let :soft_high do
+    35
+  end
+
+  let :soft_low do
+    3
+  end
+
+  let :hard_high do
+    100
+  end
+
+  let :hard_low do
+    -10
+  end
+
+  let! :starting_time do
+    Time.now
+  end
+
+  def time_adjust(millis)
+    Time.stub!(:now).and_return(Time.at(starting_time, millis))
+  end
+
+  let :autoscaler do
+    time_adjust(0)
+    OurEelHacks::Autoscaler.new.tap do |test|
+      test.configure do |test|
+        test.app_name = app_name
+        test.heroku_api_key = api_key
+        test.ps_type = "web"
+        test.scaling_frequency = scaling_freq
+        test.soft_duration = soft_dur
+
+        test.lower_limits.hard = 1
+        test.lower_limits.soft = 5
+
+        test.upper_limits.soft = 30
+        test.upper_limits.hard = 50
+      end
+    end
+  end
+
+  it "should get a count of dynos at start" do
+    autoscaler.dynos.should == 3 #happens to be the number of web dynos right now
+  end
+
+  before :each do
+    autoscaler.stub!(:set_dynos)
+    time_adjust(0)
+    autoscaler.scale(ideal_value)
+  end
+
+  describe "scaling frequency" do
+
+    it "should not scale too soon" do
+      time_adjust(scaling_freq - 5)
+
+      autoscaler.should_not_receive(:set_dynos)
+      autoscaler.scale(hard_high)
+    end
+
+    it "should scale up if time has elapsed and hard limit exceeded" do
+      time_adjust(scaling_freq + 5)
+
+      autoscaler.should_receive(:set_dynos).with(4)
+      autoscaler.scale(hard_high)
+    end
+  end
+
+  describe "hard limits" do
+    before :each do
+      time_adjust(scaling_freq + 5)
+    end
+
+    it "should scale down if hard lower limit exceeded" do
+      autoscaler.should_receive(:set_dynos).with(2)
+      autoscaler.scale(hard_low)
+    end
+  end
+
+  describe "soft upper limit" do
+    before :each do
+      time_adjust(scaling_freq * 2)
+      autoscaler.scale(soft_high)
+    end
+
+    describe "if soft_duration hasn't elapsed" do
+      before :each do
+        time_adjust((scaling_freq * 2) + soft_dur - 5)
+        autoscaler.should_receive(:set_dynos).with(3)
+      end
+
+      it "should not scale up" do
+        autoscaler.scale(soft_high)
+      end
+
+      it "should not scale down" do
+        autoscaler.scale(soft_low)
+      end
+    end
+
+    describe "if soft_duration has elapsed" do
+      before :each do
+        time_adjust(scaling_freq * 2 + soft_dur + 5)
+      end
+
+      it "should scale up if above upper soft limit" do
+        autoscaler.should_receive(:set_dynos).with(4)
+        autoscaler.scale(soft_high)
+      end
+
+      it "should not scale down if below lower soft limit" do
+        autoscaler.should_receive(:set_dynos).with(3)
+        autoscaler.scale(soft_low)
+      end
+    end
+  end
+
+  describe "soft lower limit" do
+    before :each do
+      time_adjust(scaling_freq * 2)
+      autoscaler.scale(soft_low)
+    end
+
+    describe "if soft_duration hasn't elapsed" do
+      before :each do
+        time_adjust(scaling_freq * 2 + soft_dur - 5)
+        autoscaler.should_receive(:set_dynos).with(3)
+      end
+
+      it "should not scale up" do
+        autoscaler.scale(soft_high)
+      end
+
+      it "should not scale down" do
+        autoscaler.scale(soft_low)
+      end
+    end
+
+    describe "if soft_duration has elapsed" do
+      before :each do
+        time_adjust(scaling_freq * 2 + soft_dur + 5)
+      end
+
+      it "should not scale up even if above upper soft limit" do
+        autoscaler.should_receive(:set_dynos).with(3)
+        autoscaler.scale(soft_high)
+      end
+
+      it "should scale down if below lower soft limit" do
+        autoscaler.should_receive(:set_dynos).with(2)
+        autoscaler.scale(soft_low)
+      end
+    end
+  end
+end

data/spec/rack.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'our-eel-hacks/rack'
+
+describe OurEelHacks::Rack do
+  use_vcr_cassette :record => :once
+
+  let :app_name do
+    "sbmp"
+  end
+
+  let :api_key do
+    "FakeApiKey"
+  end
+
+  before :each do
+    OurEelHacks::Autoscaler.configure(:test) do |test|
+      test.app_name = app_name
+      test.heroku_api_key = api_key
+      test.ps_type = "web"
+    end
+  end
+
+  let :fake_app do
+    mock("Rack App").tap do |app|
+      app.stub!(:call)
+    end
+  end
+
+  let :env_field do
+    "HTTP_X_HEROKU_QUEUE_DEPTH"
+  end
+
+  let :middleware do
+    OurEelHacks::Rack.new(fake_app, env_field, :test)
+  end
+
+
+  it "should pass the metric to the autoscaler" do
+    OurEelHacks::Autoscaler.instance_for(:test).should_receive(:scale).with(100)
+    middleware.call({env_field => "100"})
+  end
+end