alicorn 0.0.4 → 0.0.5

data/VERSION

@@ -1 +1 @@
-0.0.4
+0.0.5

data/alicorn.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = "alicorn"
-  s.version = "0.0.4"
+  s.version = "0.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Ben Somers"]
-  s.date = "2012-05-11"
+  s.date = "2012-05-15"
   s.description = "Highly configurable dumb auto-scaler for managing unicorn web servers"
   s.email = "somers.ben@gmail.com"
-  s.executables = ["alicorn"]
+  s.executables = ["alicorn", "alicorn_profiler"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.rdoc"
@@ -27,8 +27,10 @@ Gem::Specification.new do |s|
     "VERSION",
     "alicorn.gemspec",
     "bin/alicorn",
+    "bin/alicorn_profiler",
     "lib/alicorn.rb",
     "lib/alicorn/dataset.rb",
+    "lib/alicorn/log_parser.rb",
     "lib/alicorn/scaler.rb",
     "test/helper.rb",
     "test/test_alicorn.rb"

data/bin/alicorn

@@ -6,48 +6,52 @@ require 'alicorn'
 options = {}
 OptionParser.new do |opts|
   opts.banner = "Alicorn is a tool for auto-scaling unicorn worker counts.
-    It relies on unicorn's built-in Raindrops middleware (so make sure you have
-    enabled it). It computes rough load scores based on separately averaged
-    data sets. , \n\nUsage: alicorn [options]"
+It relies on unicorn's built-in Raindrops middleware (so make sure you have
+enabled it). It computes rough load scores based on separately averaged
+data sets. \n\nUsage: alicorn [options]"
     opts.on( '-h', '--help', 'Display this screen' ) do
       puts opts
       exit
     end
 
-  opts.on("--min-workers N", Integer, "the minimum number of workers to scale down to") do |v|
+  opts.on("--min-workers [N]", Integer, "the minimum number of workers to scale down to") do |v|
     options[:min_workers] = v
   end
 
-  opts.on("--max-workers N", Integer, "the maximum number of workers to scale up to") do |v|
+  opts.on("--max-workers [N]", Integer, "the maximum number of workers to scale up to") do |v|
     options[:max_workers] = v
   end
 
-  opts.on("--upward-step-size N", Integer, "the number of workers to add when scaling up") do |v|
-    options[:upward_step_size] = v
+  opts.on("--buffer [N]", Integer, "the number of extra workers to keep for safety's sake") do |v|
+    options[:buffer] = v
   end
 
-  opts.on("--downward-step-size N", Integer, "the number of workers to remove when scaling down") do |v|
-    options[:downward_step_size] = v
+  opts.on("--target-ratio [F]", Float, "the desired ratio of workers to busy workers") do |v|
+    options[:target_ratio] = v
   end
 
-  opts.on("--upward-threshold (0-1)", Float, "the proportion of busy workers to trigger an upscale") do |v|
-    options[:upward_threshold] = v
+  opts.on("--url [URL]", String, "raindrops URL to check, defaults to 127.0.0.1/_raindrops") do |v|
+    options[:url] = v
   end
 
-  opts.on("--downward-threshold (0-1)", Float, "the proportion of busy workers to trigger a downscale") do |v|
-    options[:downward_threshold] = v
+  opts.on("--sample-count [N]", Integer, "number of data points to check before scaling") do |v|
+    options[:sample_count] = v
   end
 
-  opts.on("--url S", String, "raindrops URL to check, defaults to 127.0.0.1/_raindrops") do |v|
-    options[:url] = v
+  opts.on("--delay [F]", Float, "delay between checks") do |v|
+    options[:delay] = v
   end
 
-  opts.on("--sample-count N", Integer, "number of data points to check before scaling") do |v|
-    options[:sample_count] = v
+  opts.on("--log-path [PATH]", String, "path to write log file - leave blank for no logging") do |v|
+    options[:log_path] = v
   end
 
-  opts.on("--delay (seconds)", Float, "delay between checks") do |v|
-    options[:delay] = v
+  opts.on("-v", "--[no-]verbose", "turn on to write profiling info") do
+    options[:verbose] = true
+  end
+
+  opts.on("-d", "--[no-]dry-run", "turn on to disable actual scaling") do
+    options[:dry_run] = true
   end
 
 end.parse!

data/bin/alicorn_profiler

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+$:.push File.join(File.dirname(__FILE__),'..','lib')
+
+require 'optparse'
+require 'alicorn'
+require 'alicorn/log_parser'
+
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Alicorn Profiler is a profiling tool to use for determining
+optimal settings to run alicorn at. First, run the scaler in dry-run mode
+to collect statistics about your application load. Then pull that log file
+and feed it into this profiler. Experiment with the options until you find
+settings that work well for you. Be aware that occasional 'Overloaded!' 
+situations are not necessarily bad; they just result in requests getting 
+queued. \n\nUsage: alicorn [options]"
+    opts.on( '-h', '--help', 'Display this screen' ) do
+      puts opts
+      exit
+    end
+
+  opts.on("--min-workers N", Integer, "the minimum number of workers to scale down to") do |v|
+    options[:min_workers] = v
+  end
+
+  opts.on("--max-workers N", Integer, "the maximum number of workers to scale up to") do |v|
+    options[:max_workers] = v
+  end
+
+  opts.on("--buffer N", Integer, "the number of extra workers to keep for safety's sake") do |v|
+    options[:buffer] = v
+  end
+
+  opts.on("--target-ratio F", Float, "the desired ratio of workers to busy workers") do |v|
+    options[:target_ratio] = v
+  end
+
+  opts.on("--log-path PATH", String, "location of the alicorn log file to read") do |v|
+    options[:log_path] = v
+  end
+end.parse!
+
+# Use these two variables to control the profiler
+# The log file is the output of the sampling-mode alicorn
+# The options are a stripped-down option set, including only
+# the options that matter to the scaling algorithm.
+@log_file = options[:log_path]
+@options = { :min_workers => options[:min_workers],
+             :max_workers => options[:max_workers],
+             :target_ratio => options[:target_ration],
+             :buffer => options[:buffer],
+             :debug => true,
+             :dry_run => true}
+
+@alp = Alicorn::LogParser.new(@log_file)
+@alp.parse
+
+scaler = Alicorn::Scaler.new(@options)
+@worker_count = @options[:max_workers]
+
+p "Testing #{@alp.samples.count} samples"
+@alp.samples.each do |sample|
+  if sample[:active].max > @worker_count
+    p "Overloaded! Ran #{@worker_count} and got #{sample[:active].max} active"
+  end
+  sig = scaler.send(:auto_scale, sample, @worker_count)
+  @worker_count += 1 if sig == "TTIN"
+  @worker_count -= 1 if sig == "TTOU"
+end

data/lib/alicorn/dataset.rb

@@ -1,14 +1,17 @@
 class DataSet < Array
   def avg
+    return nil if empty?
     inject(:+) / size.to_i
   end
 
   def variance
+    return nil if empty?
     sum=self.inject(0){|acc,i|acc +(i-avg)**2}
     return(1/self.length.to_f*sum)
   end
 
   def stddev
+    return nil if empty?
     Math.sqrt(self.variance)
   end
 end

data/lib/alicorn/log_parser.rb

@@ -0,0 +1,36 @@
+module Alicorn
+  class LogParser
+    attr_accessor :filename, :samples, :calling, :active, :queued, :calling_avg, :active_avg, :queued_avg
+
+    def initialize(file = "alicorn.log")
+      self.filename = file
+      self.samples, self.calling, self.active, self.queued, self.calling_avg, self.active_avg, self.queued_avg = [], [], [], [], [], [], []
+    end
+    
+    def parse
+      f = File.open(filename)
+      f.each do |line|
+        if line.match(/^"Sampling/)
+          @sample_hash = {} # this will reset every sample
+        elsif line.match(/^"calling:\[(.+)\]"/)
+          data = $1.split(", ").map(&:to_i)
+          calling << data
+          @sample_hash[:calling] = data
+        elsif line.match(/^"calling avg:([\d]+)"/)
+          data = $1.to_i
+          calling_avg << data
+          @sample_hash[:calling_avg] = data
+        elsif line.match(/^"active:\[(.+)\]"/)
+          data = $1.split(", ").map(&:to_i)
+          active << data
+          @sample_hash[:active] = data
+        elsif line.match(/^"active avg:([\d]+)"/)
+          data = $1.to_i
+          active_avg << data
+          @sample_hash[:active_avg] = data
+          samples << @sample_hash
+        end
+      end
+    end
+  end
+end

data/lib/alicorn/scaler.rb

@@ -1,86 +1,103 @@
 require 'curl'
+require 'logger'
 require 'alicorn/dataset'
 
-class Alicorn::Scaler
-  attr_accessor :min_workers, :max_workers, :upward_step_size, 
-    :downward_step_size, :upward_threshold, :downward_threshold, :raindrops_url, 
-    :delay, :sample_count, :app_name, :master_pid, :worker_count, :calling, 
-    :writing, :active, :queued
+module Alicorn
+  class Scaler
+    attr_accessor :min_workers, :max_workers, :target_ratio, :buffer,
+      :raindrops_url, :delay, :sample_count, :app_name, :dry_run,
+      :master_pid, :worker_count, :logger
 
-  def initialize(options)
-    self.min_workers        = options[:min_workers]         || 1
-    self.max_workers        = options[:max_workers]
-    self.upward_threshold   = options[:upward_threshold]
-    self.downward_threshold = options[:downward_threshold]
-    self.upward_step_size   = options[:upward_step_size]    || 1
-    self.downward_step_size = options[:downward_step_size]  || 1
-    self.raindrops_url      = options[:url]                 || "http://127.0.0.1/_raindrops"
-    self.delay              = options[:delay]               || 1
-    self.sample_count       = options[:sample_count]        || 30
-    self.app_name           = options[:app_name]            || "unicorn"
-    self.master_pid         = find_master_pid(app_name)
-    self.worker_count       = find_worker_count(app_name)
-    self.calling            = DataSet.new
-    self.writing            = DataSet.new
-    self.active             = DataSet.new
-    self.queued             = DataSet.new
-  end
+    def initialize(options)
+      self.min_workers        = options[:min_workers]         || 1
+      self.max_workers        = options[:max_workers]
+      self.target_ratio       = options[:target_ratio]        || 1.3
+      self.buffer             = options[:buffer]              || 0
+      self.raindrops_url      = options[:url]                 || "http://127.0.0.1/_raindrops"
+      self.delay              = options[:delay]               || 1
+      self.sample_count       = options[:sample_count]        || 30
+      self.app_name           = options[:app_name]            || "unicorn"
+      self.dry_run            = options[:dry_run]
+      log_path                = options[:log_path]            || "/dev/null"
 
-  def scale!
-    collect_data
-    upscale or downscale
-  end
+      self.logger = Logger.new(log_path)
+      logger.level = options[:verbose] ? Logger::DEBUG : Logger::WARN
+    end
 
-protected
+    def scale!
+      master_pid    = find_master_pid
+      worker_count  = find_worker_count
+      data          = collect_data
 
-  # planned algorithm: maintain worker count at 1.3*active
-  def upscale
-    # return false if worker_count >= max_workers
-    # return false if calling.all? { |sample| sample == 0 }
-  end
+      sig = auto_scale(data, worker_count)
+      send_signal(sig) if sig
+    end
 
-  def downscale
-    # return false if worker_count <= min_workers
-  end
+  protected
 
-private
+    def auto_scale(data, worker_count)
+      # Calculate target
+      target = data[:active].max * target_ratio + buffer
 
-  def find_master_pid(app_name)
-  end
+      # Check hard thresholds
+      target = max_workers if max_workers and target > max_workers
+      target = min_workers if target < min_workers
+      target = target.ceil
 
-  def find_worker_count(app_name)
-  end
+      logger.debug "target calculated at: #{target}, worker count at #{worker_count}"
+      if target > worker_count
+        logger.debug "scaling up!" unless dry_run
+        return "TTIN"
+      elsif target < worker_count
+        logger.debug "scaling down!" unless dry_run
+        return "TTOU"
+      end
+    end
 
-  def collect_data
-    p "Sampling #{sample_count} times"
-    sample_count.times do
-      raindrops = get_raindrops(raindrops_url)
-      calling << $1.to_i if raindrops.detect { |line| line.match(/calling: ([0-9]+)/) }
-      writing << $1.to_i if raindrops.detect { |line| line.match(/writing: ([0-9]+)/) }
-      active  << $1.to_i if raindrops.detect { |line| line.match(/active: ([0-9]+)/) }
-      queued  << $1.to_i if raindrops.detect { |line| line.match(/queued: ([0-9]+)/) }
-      sleep(delay)
+    def find_master_pid
     end
-    p "Collected:"
-    p "calling:#{calling}"
-    p "calling avg:#{calling.avg}"
-    p "calling stddev:#{calling.stddev}"
-    p "writing:#{writing}"
-    p "writing avg:#{writing.avg}"
-    p "writing stddev:#{writing.stddev}"
-    p "active:#{active}"
-    p "active avg:#{active.avg}"
-    p "active stddev:#{active.stddev}"
-    p "queued:#{queued}"
-    p "queued avg:#{queued.avg}"
-    p "queued stddev:#{queued.stddev}"
-  end
 
-  def get_raindrops(url)
-    Curl::Easy.http_get(url).body_str.split("\n")
-  end
-  
-  def send_signal(sig)
-    Process.kill(sig, master_pid)
+    def find_worker_count
+      14
+    end
+
+  private
+
+    def collect_data
+      logger.debug "Sampling #{sample_count} times at #{Time.now}"
+      calling, writing, active, queued = DataSet.new, DataSet.new, DataSet.new, DataSet.new
+      sample_count.times do
+        raindrops = get_raindrops(raindrops_url)
+        calling << $1.to_i if raindrops.detect { |line| line.match(/calling: ([0-9]+)/) }
+        writing << $1.to_i if raindrops.detect { |line| line.match(/writing: ([0-9]+)/) }
+        active  << $1.to_i if raindrops.detect { |line| line.match(/active: ([0-9]+)/) }
+        queued  << $1.to_i if raindrops.detect { |line| line.match(/queued: ([0-9]+)/) }
+        sleep(delay)
+      end
+      logger.debug "Collected:"
+      logger.debug "calling:#{calling}"
+      logger.debug "calling avg:#{calling.avg}"
+      logger.debug "calling stddev:#{calling.stddev}"
+      logger.debug "writing:#{writing}"
+      logger.debug "writing avg:#{writing.avg}"
+      logger.debug "writing stddev:#{writing.stddev}"
+      logger.debug "active:#{active}"
+      logger.debug "active avg:#{active.avg}"
+      logger.debug "active stddev:#{active.stddev}"
+      logger.debug "queued:#{queued}"
+      logger.debug "queued avg:#{queued.avg}"
+      logger.debug "queued stddev:#{queued.stddev}"
+      {:calling => calling, :writing => writing, :active => active, :queued => queued}
+    end
+
+    def get_raindrops(url)
+      Curl::Easy.http_get(url).body_str.split("\n")
+    end
+    
+    def send_signal(sig)
+      return false if dry_run
+      return sig
+      # Process.kill(sig, master_pid)
+    end
   end
 end

data/test/helper.rb

@@ -12,7 +12,8 @@ require 'shoulda'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'unicorn_rider'
+require 'alicorn'
+require 'script/al'
 
 class Test::Unit::TestCase
 end

data/test/test_alicorn.rb

@@ -1,7 +1,9 @@
 require 'helper'
 
-class TestUnicornRider < Test::Unit::TestCase
+class TestScaler < Test::Unit::TestCase
+
   should "probably rename this file and start testing for real" do
     flunk "hey buddy, you should probably rename this file and start testing for real"
   end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: alicorn
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-11 00:00:00.000000000 Z
+date: 2012-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: curb
-  requirement: &70142969894040 !ruby/object:Gem::Requirement
+  requirement: &70095244082940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.8.0
   type: :runtime
   prerelease: false
-  version_requirements: *70142969894040
+  version_requirements: *70095244082940
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &70142969893260 !ruby/object:Gem::Requirement
+  requirement: &70095244082440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 3.0.1
   type: :development
   prerelease: false
-  version_requirements: *70142969893260
+  version_requirements: *70095244082440
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &70142969892540 !ruby/object:Gem::Requirement
+  requirement: &70095244081720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '3.12'
   type: :development
   prerelease: false
-  version_requirements: *70142969892540
+  version_requirements: *70095244081720
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70142969891780 !ruby/object:Gem::Requirement
+  requirement: &70095244081000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.1.3
   type: :development
   prerelease: false
-  version_requirements: *70142969891780
+  version_requirements: *70095244081000
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70142969891080 !ruby/object:Gem::Requirement
+  requirement: &70095244080240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,11 +65,12 @@ dependencies:
         version: 1.8.3
   type: :development
   prerelease: false
-  version_requirements: *70142969891080
+  version_requirements: *70095244080240
 description: Highly configurable dumb auto-scaler for managing unicorn web servers
 email: somers.ben@gmail.com
 executables:
 - alicorn
+- alicorn_profiler
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
@@ -84,8 +85,10 @@ files:
 - VERSION
 - alicorn.gemspec
 - bin/alicorn
+- bin/alicorn_profiler
 - lib/alicorn.rb
 - lib/alicorn/dataset.rb
+- lib/alicorn/log_parser.rb
 - lib/alicorn/scaler.rb
 - test/helper.rb
 - test/test_alicorn.rb
@@ -104,7 +107,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2272190929466153493
+      hash: -2775814352740060929
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: