abcrunch 0.0.4

data/.gitignore

@@ -0,0 +1,6 @@
+.idea
+.idea/*
+log
+tmp
+bin/*
+.DS_Store

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.2-p290@abcrunch --create

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in abcrunch.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,28 @@
+PATH
+  remote: .
+  specs:
+    abcrunch (0.0.4)
+      colorize
+      rr
+      rspec
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    colorize (0.5.8)
+    diff-lcs (1.1.3)
+    rr (1.0.4)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  abcrunch!

data/README

@@ -0,0 +1,176 @@
+Ab Crunch
+
+The idea behind Ab Crunch is that basic performance metrics and standards should
+be effortless, first-class citizens in the development process, with frequent visibility
+and immediate feedback when performance issues are introduced.
+
+Other tools exist for measuring performance, but we found that they had some drawbacks:
+    - Not easily integrated into routine development practices, such as automated testing and CI
+    - Take a long time to set up.
+    - Take a long time to use.
+
+We wanted a tool that, while simple, was valid enough to surface basic performance
+issues and fast/easy enough to use throughout all our projects.
+
+Ab Crunch uses Apache Bench to run various strategies for load testing web sites.
+It generates rake tasks for running all or some of our tests.  These can be configured
+to be just informational, or to fail when specified standards are not met.  The rake
+tasks can then be added to our Continuous Integration (CI) builds, so builds fail when
+performance degrades.
+
+
+Credits
+    Christopher "Kai" Lichti, Author
+    Aaron Hopkins, adviser / contributed strategies
+    John Williams, adviser / contributed strategies
+
+
+Prerequisites
+
+    Must have Apache Bench installed and 'ab' on your path
+
+
+Quick Start Guide
+
+    To see some immediate action, require the gem, and run 'rake abcrunch:example'
+
+    Now to use it on your own pages:
+
+    First, define the pages you want to test, and (optionally), the performance
+    requirements you want them to meet.  If you exclude any requirements, your
+    load test will be informational only, and won't log or raise any errors
+    based on performance standards.
+
+    For Example:
+
+      @load_test_page_sets = {
+        :production => [
+          {
+            :name => "Google home page",
+            :url => "http://www.google.com/",
+            :min_queries_per_second => 20,
+            :max_avg_response_time => 1000,
+          },
+          {
+            :name => "Facebook home page",
+            :url => "http://www.facebook.com/",
+          }
+        ],
+        :staging => [
+          {
+            :name => "Github home page",
+            :url => "http://www.github.com/",
+            :max_avg_response_time => 1000,
+          }
+        ]
+      }
+
+      require 'abcrunch'
+      AbCrunch::Config.page_sets = @load_test_page_sets
+
+    In Rails, you can do this in your development and test environments.
+
+    Once you've configured Ab Crunch, you can run rake tasks to load test your pages, like this:
+
+      rake abcrunch:staging
+       - or -
+      rake abcrunch:all
+
+
+Configuring Pages
+
+    :name  - (required) User-friendly name for the page.
+    :url  - (required) Url to test. Can be a string or a Proc.  Proc example:
+          :url => proc do
+            "http://www.google.com/?q=#{['food','coma','weirds','code'][rand(4)]}"
+          end,
+
+    Performance requirements (will raise so CI builds break when requirements fail)
+        :min_queries_per_second  - page must support at least this many QPS
+        :max_avg_response_time  - latency for the page cannot go higher than this
+
+    Other Options
+        :num_requests  - how many requests to make during each (of many) runs [Default: 50]
+        :max_latency  - global maximum latency (in ms) considered to be acceptable [Default: 1000]
+        :max_degradation_percent  - global max percent latency can degrade before being considered unacceptable [Default: 0.5 (iow 50%)]
+
+
+Example: Iterative Optimization
+  Running a focus load test to iterate fixing a performance issue.
+
+    If a specific page is too slow, you can iterate on that page using a focus rake task, like so:
+
+        rake abcrunch:dev:focus[3]
+
+
+Example: Configuring the same URLS in multiple environments (dev, qa, staging, prod...)
+
+    Here's an example showing how you might dry up the AbCrunch configuration to support multiple environments.
+
+    def init_env
+      if ['development', 'test'].include? RAILS_ENV
+        require 'abcrunch'
+        AbCrunch::Config.page_sets = ab_crunch_page_sets
+      end
+    end
+
+    def ab_crunch_page_sets
+      def page_with_domain(page, domain)
+        new = page.clone
+        new[:url] = page[:url].gsub '<domain>', domain
+        new
+      end
+
+      result = {
+        :dev => AB_CRUNCH_PAGE_SET_TEMPLATE.collect { |page| page_with_domain(page, 'http://localhost:3000') },
+        :qa => AB_CRUNCH_PAGE_SET_TEMPLATE.collect { |page| page_with_domain(page, 'http://qa.myapp.com') },
+        :staging => AB_CRUNCH_PAGE_SET_TEMPLATE.collect { |page| page_with_domain(page, 'http://staging.myapp.com') },
+        :prod => AB_CRUNCH_PAGE_SET_TEMPLATE.collect { |page| page_with_domain(page, 'http://www.myapp.com') },
+      }
+
+      result
+    end
+
+    AB_CRUNCH_PAGE_SET_TEMPLATE = [
+      {
+        :name => "Home",
+        :url => "<domain>/",
+        :min_queries_per_second => 50,
+      },
+      {
+        :name => "Blog",
+        :url => "<domain>/blog?user=honest_auto",
+        :max_avg_response_time => 450,
+      }
+    ]
+
+
+KNOWN GOTCHA: Apache Bench does not like urls that just end with the domain.  For example:
+    http://www.google.com    is BAD, but
+    http://www.google.com/   is fine, for reasons surpassing understanding.
+    ...so for root level urls, be sure to add a trailing slash.
+
+<<-LICENSE
+
+The MIT License
+Copyright (c) 2011 TrueCar, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+LICENSE

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+Dir[File.join(File.dirname(__FILE__), "lib/**/*.rb")].each { |f| require f }
+
+Dir[File.join(File.dirname(__FILE__), "lib/**/*.rake")].
+  concat(Dir[File.join(File.dirname(__FILE__), "spec/tasks/**/*.rake")]).
+  concat(Dir[File.join(File.dirname(__FILE__), "tasks/**/*.rake")]).
+  concat(Dir[File.join(File.dirname(__FILE__), "{test,spec}/*.rake")]).each  { |rake| load(rake) }
+
+task default: [:spec]

data/ab_honk.rb

@@ -0,0 +1,104 @@
+test_pages = [
+  {:name => 'Price Drop Widget', :url => 'http://widgets.qa.honk.com/wsj/price-drop?pag_id=179'},
+  {:name => 'Histogram Widget', :url => 'http://widgets.qa.honk.com/wsj/histogram?pag_id=179&make=toyota&model=corolla&year=2010&body_type=sedan'},
+  {:name => 'ES Hybrids Widget', :url => 'http://widgets.qa.honk.com/wsj/search?preset=hybrids&zip=95101'},
+  {:name => 'Makes list page', :url => 'https://staging.honk.com/wsj/makes'},
+  {:name => 'Landing Page', :url => 'https://staging.honk.com/wsj/new-cars'},
+  {:name => 'Trends Page', :url => 'https://staging.honk.com/wsj/new-car-trends'},
+  {:name => 'Explore Search Tool', :url => 'https://staging.honk.com/wsj/explore'},
+  {:name => 'Make Landing Page', :url => 'https://staging.honk.com/wsj/toyota'},
+  {:name => 'Model Landing Page', :url => 'https://staging.honk.com/wsj/toyota/corolla/sedan/2010/4dr-sedan-man'},
+  {:name => 'Model Reviews Page', :url => 'https://staging.honk.com/wsj/toyota/corolla/sedan/2010/4dr-sedan-man/reviews'},
+  {:name => 'Model Price Page', :url => 'https://staging.honk.com/wsj/toyota/corolla/sedan/2010/4dr-sedan-man/price'},
+  {:name => 'Model Specs Page', :url => 'https://staging.honk.com/wsj/toyota/corolla/sedan/2010/4dr-sedan-man/tech_specs'},
+  {:name => 'Model Photos Page', :url => 'https://staging.honk.com/wsj/toyota/corolla/sedan/2010/4dr-sedan-man/media'}
+]
+
+def parse_ab_data(ab_log)
+  {
+    :response_time => ab_log.match(/Time per request:\s*([\d\.]+)\s\[ms\]\s\(mean\)/)[1].to_f,
+    :queries_per_second => ab_log.match(/Requests per second:\s*([\d\.]+)\s\[#\/sec\]\s\(mean\)/)[1].to_f,
+    :failed_requests => ab_log.match(/Failed requests:\s*([\d\.]+)/)[1].to_f
+  }
+end
+
+def ab(url, concurrency, num_requests)
+  `ab -c #{concurrency} -n #{num_requests} #{url}`
+end
+
+def best_of(url, concurrency, num_requests, num_runs)
+  min_response_time = 999999
+  min_response_log = ''
+  puts "\nBest of #{num_runs} at concurrency: #{concurrency} and num_requests: #{num_requests}"
+  puts "Collecting average response times for each run:"
+  num_runs.times do
+    ab_log = ab(url, concurrency, num_requests)
+    response_time = parse_ab_data(ab_log)[:response_time]
+    print "#{response_time} ... "
+    STDOUT.flush
+    if response_time < min_response_time
+      min_response_time = response_time
+      min_response_log = ab_log
+    end
+  end
+  [min_response_time, min_response_log]
+end
+
+def baseline_page(page)
+  puts "\nCalculating Baseline (min average response time over multiple runs)"
+  best_of(page[:url], 1, 10, 5)
+end
+
+def find_max_concurrency(page, baseline_response_time, baseline_log, threshold_percent)
+  puts "\nFinding the max concurrency without degrading performance beyond a threshold"
+  threshold_ms = [1000.0, baseline_response_time * (1 + threshold_percent)].min
+  puts "Threshold: #{threshold_ms} (ms)"
+  print "Trying ever increasing concurrency until we bust the threshold"
+  concurrency = 0
+  ab_log = baseline_log
+  begin
+    concurrency += 1
+    prev_log = ab_log
+    response_time, ab_log = best_of(page[:url], concurrency, 10, 3)
+  end while response_time < threshold_ms
+  [concurrency - 1, threshold_ms, prev_log]
+end
+
+test_pages.each do |page|
+  puts "\n\n\n=========== Testing #{page[:name]}..."
+
+  baseline_response_time, baseline_output = baseline_page(page)
+  page[:baseline_response_time] = baseline_response_time
+  puts "\n----- Min Average Response Time was #{baseline_response_time}"
+  #puts "----- Matching AB Output"
+  #puts baseline_output
+
+  max_concurrency, threshold, mc_output = find_max_concurrency(page, baseline_response_time, baseline_output, 0.5)
+  page[:max_concurrency] = max_concurrency
+  page[:threshold] = threshold
+  page[:queries_per_second] = parse_ab_data(mc_output)[:queries_per_second]
+  puts "\n----- Max Concurrency was #{max_concurrency}"
+  puts "----- Threshold was #{threshold}"
+  puts "----- Matching AB Output at max concurrency"
+  puts mc_output
+end
+
+class Float
+  alias_method :orig_to_s, :to_s
+  def to_s(arg = nil)
+    if arg.nil?
+      orig_to_s
+    else
+      sprintf("%.#{arg}f", self)
+    end
+  end
+end
+
+puts "\nSummary"
+puts "#{"Page".ljust(30, ' ')}#{"Baseline".rjust(10, ' ')}  #{"Max Concurrency".rjust(16, ' ')}  #{"Threshold".rjust(10, ' ')}  #{"Queries/sec".rjust(12, ' ')}"
+test_pages.each do |page|
+  puts "#{page[:name].ljust(30, ' ')}#{page[:baseline_response_time].to_s(2).rjust(10, ' ')}  #{page[:max_concurrency].to_s.rjust(16, ' ')}  #{page[:threshold].to_s(0).rjust(10, ' ')}  #{page[:queries_per_second].to_s(2).rjust(12, ' ')}"
+end
+puts "\nBaseline = Best average response time (ms) over multiple runs with no concurrency"
+puts "Max Concurrency = Most concurrent requests where best average response time doesn't bust our performance threshold"
+puts "Queries/sec = Queries per second at max concurrency"

data/abcrunch.gemspec

@@ -0,0 +1,36 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "abcrunch/version"
+
+Gem::Specification.new do |s|
+  s.name        = "abcrunch"
+  s.version     = AbCrunch::VERSION
+  s.authors     = ["Chris Lichti"]
+  s.email       = ["kai@truecar.com"]
+  s.homepage    = "https://github.com/kurisu/abcrunch"
+  s.summary     = "Automated load testing in ruby"
+  s.description = <<-DESC
+The idea behind Ab Crunch is that basic performance metrics and standards should
+be effortless, first-class citizens in the development process, with frequent visibility
+and immediate feedback when performance issues are introduced.
+
+Ab Crunch uses Apache Bench to run various strategies for load testing web projects,
+and provides rake tasks for analyzing performance and enforcing performance
+standards on CI.
+DESC
+
+  s.rubyforge_project = "abcrunch"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "rspec"
+  s.add_runtime_dependency "rr"
+  s.add_runtime_dependency "colorize"
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

data/lib/abcrunch.rb

@@ -0,0 +1,11 @@
+require "abcrunch/version"
+require 'rake'
+
+Dir[File.join(File.dirname(__FILE__), "**/*.rb")].each { |f| require f }
+Dir[File.join(File.dirname(__FILE__), "**/*.rake")].each  { |rake| load(rake) }
+
+module AbCrunch
+  def self.root
+    File.dirname(File.dirname(__FILE__))
+  end
+end

data/lib/abcrunch/ab_result.rb

@@ -0,0 +1,33 @@
+module AbCrunch
+  class AbResult
+    attr_accessor :raw, :ab_options
+
+    def initialize(raw_ab_output, ab_options)
+      @raw = raw_ab_output
+      @ab_options = ab_options
+    end
+
+    def command
+      AbCrunch::AbRunner.ab_command(@ab_options)
+    end
+
+    def avg_response_time
+      raw.match(/Time per request:\s*([\d\.]+)\s\[ms\]\s\(mean\)/)[1].to_f
+    end
+
+    def queries_per_second
+      raw.match(/Requests per second:\s*([\d\.]+)\s\[#\/sec\]\s\(mean\)/)[1].to_f
+    end
+
+    def failed_requests
+      raw.match(/Failed requests:\s*([\d\.]+)/)[1].to_i
+    end
+
+    def log
+      AbCrunch::Logger.log :ab_result, "#{command}"
+      AbCrunch::Logger.log :ab_result, "Average Response Time: #{avg_response_time}"
+      AbCrunch::Logger.log :ab_result, "Queries per Second: #{queries_per_second}"
+      AbCrunch::Logger.log :ab_result, "Failed requests: #{failed_requests}"
+    end
+  end
+end

data/lib/abcrunch/ab_runner.rb

@@ -0,0 +1,36 @@
+require 'open3'
+require 'pp'
+
+module AbCrunch
+  class AbRunner
+    def self.validate_options(options)
+      raise "AB Options missing :url" unless options.has_key? :url
+      AbCrunch::Config.ab_options.merge(options)
+    end
+
+    def self.ab_command(options)
+      options = validate_options(options)
+      url = AbCrunch::Page.get_url(options, true)
+      AbCrunch::Logger.log(:info, "Calling ab on:  #{url}")
+      "ab -c #{options[:concurrency]} -n #{options[:num_requests]} -k -H 'Accept-Encoding: gzip' #{url}"
+    end
+
+    def self.ab(options)
+      cmd = ab_command(options)
+      result = nil
+
+      Open3.popen3(cmd) do |stdin, stdout, stderr|
+        err_lines = stderr.readlines
+        if err_lines.length > 0
+          cmd = cmd.cyan
+          err = "AB command failed".red
+          err_s = err_lines.reduce {|line, memo| memo += line}.red
+          raise "#{err}\nCommand: #{cmd}\n#{err_s}"
+        end
+        result = AbCrunch::AbResult.new stdout.readlines.reduce {|line, memo| memo += line}, options
+      end
+
+      result
+    end
+  end
+end