abcrunch 0.0.4

data/lib/abcrunch/best_run.rb

@@ -0,0 +1,22 @@
+module AbCrunch
+  class BestRun
+    def self.of_avg_response_time(num_runs, ab_options)
+      AbCrunch::Logger.log :task, "Best of #{num_runs} runs at concurrency: #{ab_options[:concurrency]} and num_requests: #{ab_options[:num_requests]}"
+      AbCrunch::Logger.log :info, "for #{AbCrunch::Page.get_url(ab_options)}"
+      AbCrunch::Logger.log :info, "Collecting average response times for each run:"
+
+      min_response_time = 999999
+      min_response_result = nil
+      num_runs.times do
+        abr = AbCrunch::AbRunner.ab(ab_options)
+        AbCrunch::Logger.log :info, "Average response time: #{abr.avg_response_time} (ms) "
+        if abr.avg_response_time < min_response_time
+          min_response_time = abr.avg_response_time
+          min_response_result = abr
+        end
+      end
+      AbCrunch::Logger.log :progress, "Best response time was #{min_response_time}"
+      min_response_result
+    end
+  end
+end

data/lib/abcrunch/config.rb

@@ -0,0 +1,35 @@
+module AbCrunch
+  module Config
+    class << self
+      attr_accessor :page_sets, :best_concurrency_options, :ab_options
+    end
+
+    @best_concurrency_options = {
+      :max_degradation_percent => 0.5,
+      :max_latency => 1000.0,
+      :num_baseline_runs => 5,
+      :num_concurrency_runs => 3
+    }
+
+    @ab_options = {
+      :concurrency => 1,
+      :num_requests => 100
+    }
+
+    @page_sets = {
+      :localhost => [
+        {
+          :name => 'localhost',
+          :url => 'http://localhost:3000/',
+          :max_avg_response_time => 500
+        }
+      ]
+    }
+
+    def self.page_sets=(new_page_sets)
+      @page_sets = new_page_sets
+      require 'rake'
+      load File.join(AbCrunch.root, "lib/abcrunch/tasks/generated.rake")
+    end
+  end
+end

data/lib/abcrunch/log_console_writer.rb

@@ -0,0 +1,44 @@
+require 'colorize'
+
+module AbCrunch
+  class LogConsoleWriter
+    @@last_inline = false
+
+    TYPE_STYLES = {
+      :info => { :color => :white, :prefix => '  ' },
+      :test => { :color => :light_white, :prefix => "\n\n-----" },
+      :strategy => { :color => :light_white, :prefix => "\n" },
+      :task => { :color => :white, :prefix => "\n " },
+      :progress => { :color => :green, :prefix => '  ' },
+      :result => { :color => :light_green, :prefix => '  ' },
+      :ab_result => { :color => :cyan, :prefix => '    ' },
+      :success => { :color => :light_green, :prefix => '  ' },
+      :failure => { :color => :light_red, :prefix => '  ' },
+      :summary => { :color => :light_white, :prefix => '' },
+      :summary_title => { :color => :light_white, :prefix => "\n==================== " },
+      :summary_passed => { :color => :light_green, :prefix => '' },
+      :summary_failed => { :color => :light_red, :prefix => '' },
+
+    }
+
+    def self.color_for_type(type)
+      TYPE_STYLES[type] ? TYPE_STYLES[type][:color] : :white
+    end
+
+    def self.prefix_for_type(type)
+      TYPE_STYLES[type] ? TYPE_STYLES[type][:prefix] : ''
+    end
+
+    def self.log(type, message, options = {})
+      a_message = prefix_for_type(type) + message
+      if options[:inline]
+        print a_message.send(color_for_type type)
+        @@last_inline = true
+      else
+        a_message = "\n#{a_message}" if @@last_inline
+        puts a_message.send(color_for_type type)
+        @@last_inline = false
+      end
+    end
+  end
+end

data/lib/abcrunch/logger.rb

@@ -0,0 +1,13 @@
+module AbCrunch
+  module Logger
+    class << self
+      attr_accessor :writers
+    end
+
+    @writers = [AbCrunch::LogConsoleWriter]
+
+    def self.log(type, message, options = {})
+      writers.each { |writer| writer.log(type, message, options) }
+    end
+  end
+end

data/lib/abcrunch/page.rb

@@ -0,0 +1,26 @@
+module AbCrunch
+  module Page
+    def self.get_url(page, force_new = false)
+      if force_new || !page[:testing_url]
+        url = page[:url]
+        if url.respond_to? :call
+          url = url.call
+        end
+        page[:testing_url] = url
+      else
+        url = page[:testing_url]
+      end
+
+      url
+    end
+
+    def self.get_display_url(page)
+      if page[:url].respond_to? :call
+        return "Dynamic url example: #{page[:url].call}"
+      else
+        page[:url]
+      end
+    end
+
+  end
+end

data/lib/abcrunch/page_tester.rb

@@ -0,0 +1,44 @@
+module AbCrunch
+  module PageTester
+    def self.test(page)
+      AbCrunch::Logger.log :test, "Testing #{page[:name]}"
+      AbCrunch::Logger.log :info, "#{AbCrunch::Page.get_display_url(page)}"
+
+      if page[:max_avg_response_time]
+        page.merge!({:max_latency => page[:max_avg_response_time]})
+      end
+      qps_result = AbCrunch::StrategyBestConcurrency.run(page)
+
+      passed = true
+      errors = []
+
+      if page[:max_avg_response_time]
+        if qps_result.avg_response_time > page[:max_avg_response_time]
+          passed = false
+          errors << "#{page[:name]}: Avg response time of #{qps_result.avg_response_time} must be <= #{page[:max_avg_response_time]}"
+        end
+      end
+
+      if page[:min_queries_per_second]
+        if qps_result.queries_per_second < page[:min_queries_per_second]
+          passed = false
+          errors << "#{page[:name]}: QPS of #{qps_result.queries_per_second} must be >= #{page[:min_queries_per_second]}"
+        end
+      end
+
+      if qps_result.failed_requests > 0
+        passed = false
+        errors << "#{page[:name]}: Load test invalidated: #{qps_result.failed_requests} requests failed"
+      end
+
+      if passed
+        AbCrunch::Logger.log :success, "PASSED"
+      else
+        errors.each { |error| AbCrunch::Logger.log :failure, error }
+        AbCrunch::Logger.log :failure, "#{page[:name]} FAILED"
+      end
+
+      [passed, qps_result, errors]
+    end
+  end
+end

data/lib/abcrunch/strategy_best_concurrency.rb

@@ -0,0 +1,48 @@
+module AbCrunch
+  class StrategyBestConcurrency
+
+    def self.calc_threshold(baseline_latency, percent_buffer, max_latency)
+      [max_latency, baseline_latency * (1 + percent_buffer)].min
+    end
+
+    def self.find_best_concurrency(page, baseline_result)
+      threshold_ms = calc_threshold(baseline_result.avg_response_time, page[:max_degradation_percent], page[:max_latency].to_f)
+
+      AbCrunch::Logger.log :task, "Finding the max concurrency without degrading performance beyond a threshold"
+      AbCrunch::Logger.log :info, "Threshold: #{threshold_ms} (ms)"
+      AbCrunch::Logger.log :info, "Trying ever increasing concurrency until we bust the threshold"
+
+      fmc_page = page.clone
+      fmc_page[:concurrency] = 0
+      abr = baseline_result
+      begin
+        fmc_page[:concurrency] += 1
+        prev_result = abr
+        abr = AbCrunch::BestRun.of_avg_response_time(fmc_page[:num_concurrency_runs], fmc_page)
+      end while abr.avg_response_time < threshold_ms and fmc_page[:concurrency] < fmc_page[:num_requests]
+      if abr.avg_response_time < threshold_ms
+        return abr || baseline_result
+      else
+        fmc_page[:concurrency] -= 1
+        return prev_result || baseline_result
+      end
+    end
+
+    def self.run(page)
+      page = AbCrunch::Config.best_concurrency_options.merge(page)
+
+      AbCrunch::Logger.log :strategy, "Strategy: find queries per second (QPS) at highest concurrency before latency degrades"
+
+      AbCrunch::Logger.log :task, "Calculating Baseline (min average response time over multiple runs)"
+      baseline_result = AbCrunch::BestRun.of_avg_response_time(page[:num_baseline_runs], page)
+
+      best_result = find_best_concurrency(page, baseline_result)
+
+      AbCrunch::Logger.log :progress, "Highest concurrency without degrading latency: #{best_result.ab_options[:concurrency]}"
+      AbCrunch::Logger.log :result, "Queries Per Second (QPS): #{best_result.queries_per_second}"
+      best_result.log
+
+      best_result
+    end
+  end
+end

data/lib/abcrunch/tasks/default.rake

@@ -0,0 +1,12 @@
+namespace :abcrunch do
+  desc "Run load tests against ALL page sets"
+  task :all do
+    if AbCrunch::Config.page_sets.length == 0
+      AbCrunch::Logger.log(:failure, "No page sets defined. Nothing to do")
+    end
+
+    AbCrunch::Config.page_sets.keys.each do |page_set_key|
+      AbCrunch::Tester.test(AbCrunch::Config.page_sets[page_set_key])
+    end
+  end
+end

data/lib/abcrunch/tasks/example.rake

@@ -0,0 +1,33 @@
+namespace :abcrunch do
+
+  desc "Test example pages"
+  task :example do
+    AbCrunch::Config.page_sets = @sample_page_sets
+    Rake::Task['abcrunch:sample_1'].invoke
+  end
+
+  @sample_page_sets = {
+    :sample_1 => [
+      {
+        :name => "Google home page",
+        :url => proc do
+          "http://www.google.com/?q=#{['food','coma','weirds','code'][rand(4)]}"
+        end,
+        :min_queries_per_second => 10,
+        :max_avg_response_time => 1000,
+      },
+      {
+        :name => "Facebook home page",
+        :url => "http://www.facebook.com/",
+        :max_avg_response_time => 1000,
+      },
+      {
+        :name => "Bing Homepage",
+        :url => "http://www.bing.com/",
+        :min_queries_per_second => 50,
+        :max_avg_response_time => 200
+      }
+    ]
+  }
+
+end

data/lib/abcrunch/tasks/generated.rake

@@ -0,0 +1,36 @@
+namespace :abcrunch do
+
+  AbCrunch::Config.page_sets.keys.each do |page_set_key|
+    unless Rake::Task.task_defined? page_set_key
+      desc "Run load tests for #{page_set_key}"
+      task page_set_key do
+        AbCrunch::Tester.test(AbCrunch::Config.page_sets[page_set_key])
+      end
+
+      namespace page_set_key do
+        desc "Run a focused page load test in #{page_set_key}. Example rake abcrunch:#{page_set_key}:focus[0] runs the first page"
+        task :focus, [:page_index] do |t, args|
+          unless verify_abcrunch_args(args, page_set_key)
+            raise "usage: rake abcrunch:#{page_set_key}:focus[<page_index>]"
+          end
+
+          orig_page = AbCrunch::Config.page_sets[page_set_key][args[:page_index].to_i]
+          AbCrunch::PageTester.test(orig_page)
+        end
+
+        def verify_abcrunch_args(args, page_set_key)
+          max_idx = AbCrunch::Config.page_sets[page_set_key].length-1
+          if !args[:page_index]
+            puts "Focusing on... NOTHING!  (dork)"
+            return false
+          elsif !(0..max_idx).include?(args[:page_index].to_i)
+            puts "Page index (#{args[:page_index]}) not in range (0..#{max_idx})"
+            return false
+          end
+          true
+        end
+      end
+    end
+  end
+
+end

data/lib/abcrunch/tester.rb

@@ -0,0 +1,56 @@
+module AbCrunch
+  module Tester
+    def self.test(pages)
+      results = []
+
+      pages.each do |page|
+        passed, qps_result, errors = AbCrunch::PageTester.test(page)
+        results << {
+          :page => page,
+          :passed => passed,
+          :qps_result => qps_result,
+          :errors => errors
+        }
+      end
+
+      log_result_summary results
+
+      passed = results.reduce(true) { |value, result| value && result[:passed] }
+
+      if not passed
+        errors = results.reduce('') do |value, result|
+          page_set_errors = result[:errors].reduce('') do |val, error|
+            "#{val}#{val.length > 0 ? "\n" : ''}#{error}"
+          end
+          "#{value}#{value.length > 0 ? "\n" : ''}#{page_set_errors}"
+        end
+
+        raise "Load tests FAILED\n#{errors}"
+      end
+
+      results
+    end
+
+    def self.log_result_summary(results)
+      AbCrunch::Logger.log :summary_title, "Summary"
+      AbCrunch::Logger.log :summary, "#{"Page".ljust(30, ' ')}#{"Response time".rjust(10, ' ')}  #{"Concurrency".rjust(16, ' ')}  #{"Queries/sec".rjust(12, ' ')}"
+      results.each do |result|
+        page_name = result[:page][:name].ljust(30, ' ')
+        base_response_time = sprintf("%.2f", result[:qps_result].avg_response_time).rjust(10, ' ')
+        max_concurrency = result[:qps_result].ab_options[:concurrency].to_s.rjust(16, ' ')
+        queries_per_second = sprintf("%.2f", result[:qps_result].queries_per_second).rjust(12, ' ')
+        if result[:passed]
+          AbCrunch::Logger.log :summary_passed, "#{page_name}#{base_response_time}  #{max_concurrency}  #{queries_per_second}"
+        else
+          AbCrunch::Logger.log :summary_failed, "#{page_name}#{base_response_time}  #{max_concurrency}  #{queries_per_second}"
+        end
+      end
+      AbCrunch::Logger.log :summary_title, "Legend"
+      AbCrunch::Logger.log :summary, "Response Time = Best average response time (ms) at max concurrency"
+      AbCrunch::Logger.log :summary, "Concurrency = Best concurrency where response time doesn't bust our performance threshold"
+      AbCrunch::Logger.log :summary, "Queries/sec = Queries per second at best concurrency"
+
+      results
+    end
+  end
+end

data/lib/abcrunch/version.rb

@@ -0,0 +1,3 @@
+module AbCrunch
+  VERSION = "0.0.4"
+end

data/spec/helpers/page_helper.rb

@@ -0,0 +1,62 @@
+module AbCrunchSpec
+
+  FULL_TEST_PAGE = {
+      :name => "some page",
+      :url => "some url",
+    }
+
+  def self.new_page(override_options = {})
+    FULL_TEST_PAGE.merge(override_options)
+  end
+
+  def self.new_result(page = FULL_TEST_PAGE)
+    AbCrunch::AbResult.new(FAKE_AB_RESULT_TEXT, page)
+  end
+
+  FAKE_AB_RESULT_TEXT = <<-ABRESULT
+This is ApacheBench, Version 2.3 <$Revision: 655654 $>
+Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
+Licensed to The Apache Software Foundation, http://www.apache.org/
+
+Benchmarking www.google.com (be patient).....done
+
+
+Server Software:        gws
+Server Hostname:        www.google.com
+Server Port:            80
+
+Document Path:          /
+Document Length:        10372 bytes
+
+Concurrency Level:      1
+Time taken for tests:   0.880 seconds
+Complete requests:      10
+Failed requests:        0
+   (Connect: 0, Receive: 0, Length: 8, Exceptions: 0)
+Write errors:           0
+Total transferred:      109542 bytes
+HTML transferred:       103762 bytes
+Requests per second:    11.36 [#/sec] (mean)
+Time per request:       88.019 [ms] (mean)
+Time per request:       88.019 [ms] (mean, across all concurrent requests)
+Transfer rate:          121.54 [Kbytes/sec] received
+
+Connection Times (ms)
+              min  mean[+/-sd] median   max
+Connect:       24   29   4.4     28      38
+Processing:    56   59   3.2     59      66
+Waiting:       52   55   3.4     55      64
+Total:         80   88   6.4     88     103
+
+Percentage of the requests served within a certain time (ms)
+  50%     88
+  66%     89
+  75%     89
+  80%     91
+  90%    103
+  95%    103
+  98%    103
+  99%    103
+ 100%    103 (longest request)
+      ABRESULT
+end