abcrunch 0.0.4

data/spec/lib/strategy_best_concurrency_spec.rb

@@ -0,0 +1,124 @@
+require 'spec_helper'
+
+describe "AbCrunch::StrategyBestConcurrency" do
+
+  describe "#run" do
+    before :each do
+      @test_page = AbCrunchSpec.new_page
+      @fake_result = AbCrunchSpec.new_result
+
+      stub(AbCrunch::BestRun).of_avg_response_time { @fake_result }
+      stub(AbCrunch::StrategyBestConcurrency).find_best_concurrency { @fake_result }
+      stub(AbCrunch::Logger).log
+    end
+
+    it "should get the baseline using the global options merged with the page" do
+      page = AbCrunch::Config.best_concurrency_options.merge(@test_page)
+      mock(AbCrunch::BestRun).of_avg_response_time(page[:num_baseline_runs], page)
+
+      AbCrunch::StrategyBestConcurrency.run(@test_page)
+    end
+
+    it "should use page option overrides" do
+      in_page = @test_page.merge({:num_baseline_runs => 17})
+      expected_page = AbCrunch::Config.best_concurrency_options.merge(in_page)
+      mock(AbCrunch::BestRun).of_avg_response_time(17, expected_page) {@fake_result}
+      proxy(AbCrunch::Config.best_concurrency_options).merge(in_page)
+
+      AbCrunch::StrategyBestConcurrency.run(in_page)
+    end
+
+    it "should find the max concurrency" do
+      expected_page = AbCrunch::Config.best_concurrency_options.merge(@test_page)
+      mock(AbCrunch::StrategyBestConcurrency).find_best_concurrency(expected_page, @fake_result) {@fake_result}
+
+      AbCrunch::StrategyBestConcurrency.run(@test_page)
+    end
+  end
+
+  describe "#calc_threshold" do
+    describe "when max latency is higher than the base response time plus the percent margin" do
+      it "should return the base response time plus the percent margin" do
+        AbCrunch::StrategyBestConcurrency.calc_threshold(100, 0.2, 200).should == 120.0
+      end
+    end
+    describe "when max latency is lower than the base response time plus the percent margin" do
+      it "should return the max latency" do
+        AbCrunch::StrategyBestConcurrency.calc_threshold(190, 0.2, 200).should == 200.0
+      end
+    end
+  end
+  
+  describe "#find_best_concurrency" do
+    before :each do
+      @test_page = AbCrunchSpec.new_page({:num_requests => 50})
+      @fake_result = AbCrunchSpec.new_result
+
+      stub(AbCrunch::Logger).log
+    end
+
+    describe "when performance degrades" do
+      it "should return the ab result for the run with the highest concurrency before response time degrades" do
+        input_page = AbCrunch::Config.best_concurrency_options.merge(@test_page).merge({:num_concurrency_runs => 3})
+        test_page_1 = input_page.clone.merge({:concurrency => 1})
+        test_result_1 = @fake_result.clone
+        test_result_1.ab_options = test_page_1
+
+        test_page_2 = input_page.clone.merge({:concurrency => 2})
+        test_result_2 = @fake_result.clone
+        test_result_2.ab_options = test_page_2
+
+        test_page_3 = input_page.clone.merge({:concurrency => 3})
+        desired_result = @fake_result.clone
+        desired_result.ab_options = test_page_3
+        stub(desired_result).avg_response_time { 90.3 }
+
+        test_page_4 = input_page.clone.merge({:concurrency => 4})
+        degraded_result = @fake_result.clone
+        degraded_result.ab_options = test_page_4
+        stub(degraded_result).avg_response_time { 9999.3 }
+
+        stub(AbCrunch::BestRun).of_avg_response_time(3, test_page_1) { test_result_1 }
+        stub(AbCrunch::BestRun).of_avg_response_time(3, test_page_2) { test_result_2 }
+        stub(AbCrunch::BestRun).of_avg_response_time(3, test_page_3) { desired_result }
+        stub(AbCrunch::BestRun).of_avg_response_time(3, test_page_4) { degraded_result }
+
+        result = AbCrunch::StrategyBestConcurrency.find_best_concurrency(input_page, @fake_result)
+
+        result.ab_options[:concurrency].should == 3
+        result.avg_response_time.should == 90.3
+        result.should == desired_result
+      end
+    end
+
+    describe "when concurrency exceeds num requests before performance degrades" do
+      it "should return the latest result" do
+        input_page = AbCrunch::Config.best_concurrency_options.merge(@test_page).merge({:num_concurrency_runs => 3, :num_requests => 3})
+        test_page_1 = input_page.clone.merge({:concurrency => 1})
+        test_result_1 = @fake_result.clone
+        test_result_1.ab_options = test_page_1
+
+        test_page_2 = input_page.clone.merge({:concurrency => 2})
+        test_result_2 = @fake_result.clone
+        test_result_2.ab_options = test_page_2
+
+        test_page_3 = input_page.clone.merge({:concurrency => 3})
+        desired_result = @fake_result.clone
+        desired_result.ab_options = test_page_3
+        stub(desired_result).avg_response_time { 90.3 }
+
+        stub(AbCrunch::BestRun).of_avg_response_time(3, test_page_1) { test_result_1 }
+        stub(AbCrunch::BestRun).of_avg_response_time(3, test_page_2) { test_result_2 }
+        stub(AbCrunch::BestRun).of_avg_response_time(3, test_page_3) { desired_result }
+
+        result = AbCrunch::StrategyBestConcurrency.find_best_concurrency(input_page, @fake_result)
+
+        result.ab_options[:concurrency].should == 3
+        result.avg_response_time.should == 90.3
+        result.should == desired_result
+      end
+    end
+
+  end
+
+end

data/spec/lib/tester_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+
+describe "AbCrunch::Tester" do
+  before :each do
+    stub(AbCrunch::Logger).log
+
+    @test_pages = [
+      AbCrunchSpec.new_page,
+      AbCrunchSpec.new_page,
+      AbCrunchSpec.new_page,
+    ]
+
+    @fake_page_results = [
+      [true, AbCrunchSpec.new_result, []],
+      [true, AbCrunchSpec.new_result, []],
+      [true, AbCrunchSpec.new_result, []],
+    ]
+
+    @fake_failed_page_results = [
+      [true, AbCrunchSpec.new_result, []],
+      [true, AbCrunchSpec.new_result, []],
+      [false, AbCrunchSpec.new_result, ['some error','another']],
+    ]
+  end
+
+  describe "#test" do
+    it "should test each of the given pages" do
+      tested_pages = []
+
+      stub(AbCrunch::PageTester).test do |page|
+        tested_pages << page
+        @fake_page_results[0]
+      end
+
+      AbCrunch::Tester.test(@test_pages)
+
+      tested_pages.should == @test_pages
+    end
+
+    it "should collect and return the results for every page" do
+      expected_test_results = @fake_page_results.each_with_index.map do |page_result, idx|
+        {
+          :page => @test_pages[idx],
+          :passed => page_result[0],
+          :qps_result => page_result[1],
+          :errors => page_result[2]
+        }
+      end
+
+      call_idx = -1
+      stub(AbCrunch::PageTester).test do
+        call_idx += 1
+        @fake_page_results[call_idx]
+      end
+
+      test_results = AbCrunch::Tester.test(@test_pages)
+
+      test_results.should == expected_test_results
+    end
+
+    it "should raise an error if there are any load test failures" do
+      call_idx = -1
+      stub(AbCrunch::PageTester).test do
+        call_idx += 1
+        @fake_failed_page_results[call_idx]
+      end
+
+      lambda do
+        AbCrunch::Tester.test(@test_pages)
+      end.should raise_error "Load tests FAILED\nsome error\nanother"
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+RSpec.configure do |conf|
+  conf.mock_with :rr
+end
+
+require File.join(File.dirname(__FILE__), "../lib/abcrunch")
+
+Dir[File.join(File.dirname(__FILE__), "helpers/**/*.rb")].each { |f| require f }

data/spec/tasks/spec.rake

@@ -0,0 +1,15 @@
+require "rspec/core/rake_task"
+
+spec_tasks = Dir['spec/*/'].map { |d| File.basename(d) } - ["javascripts"]
+
+spec_tasks.each do |folder|
+  RSpec::Core::RakeTask.new("spec:#{folder}") do |t|
+    t.pattern = "./spec/#{folder}/**/*_spec.rb"
+    t.rspec_opts = %w(-fs --color)
+  end
+end
+
+desc "Run complete application spec suite"
+task 'spec' do
+  spec_tasks.each { |f| Rake::Task["spec:#{f}"].invoke }
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: abcrunch
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+  prerelease: 
+platform: ruby
+authors:
+- Chris Lichti
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-10-25 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &2155910360 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2155910360
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: &2155909940 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2155909940
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: &2155909520 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2155909520
+description: ! 'The idea behind Ab Crunch is that basic performance metrics and standards
+  should
+
+  be effortless, first-class citizens in the development process, with frequent visibility
+
+  and immediate feedback when performance issues are introduced.
+
+
+  Ab Crunch uses Apache Bench to run various strategies for load testing web projects,
+
+  and provides rake tasks for analyzing performance and enforcing performance
+
+  standards on CI.
+
+'
+email:
+- kai@truecar.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- README
+- Rakefile
+- ab_honk.rb
+- abcrunch.gemspec
+- lib/abcrunch.rb
+- lib/abcrunch/ab_result.rb
+- lib/abcrunch/ab_runner.rb
+- lib/abcrunch/best_run.rb
+- lib/abcrunch/config.rb
+- lib/abcrunch/log_console_writer.rb
+- lib/abcrunch/logger.rb
+- lib/abcrunch/page.rb
+- lib/abcrunch/page_tester.rb
+- lib/abcrunch/strategy_best_concurrency.rb
+- lib/abcrunch/tasks/default.rake
+- lib/abcrunch/tasks/example.rake
+- lib/abcrunch/tasks/generated.rake
+- lib/abcrunch/tester.rb
+- lib/abcrunch/version.rb
+- spec/helpers/page_helper.rb
+- spec/lib/ab_result_spec.rb
+- spec/lib/ab_runner_spec.rb
+- spec/lib/best_run_spec.rb
+- spec/lib/config_spec.rb
+- spec/lib/log_console_writer_spec.rb
+- spec/lib/logger_spec.rb
+- spec/lib/page_spec.rb
+- spec/lib/page_tester_spec.rb
+- spec/lib/strategy_best_concurrency_spec.rb
+- spec/lib/tester_spec.rb
+- spec/spec_helper.rb
+- spec/tasks/spec.rake
+homepage: https://github.com/kurisu/abcrunch
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: abcrunch
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Automated load testing in ruby
+test_files:
+- spec/helpers/page_helper.rb
+- spec/lib/ab_result_spec.rb
+- spec/lib/ab_runner_spec.rb
+- spec/lib/best_run_spec.rb
+- spec/lib/config_spec.rb
+- spec/lib/log_console_writer_spec.rb
+- spec/lib/logger_spec.rb
+- spec/lib/page_spec.rb
+- spec/lib/page_tester_spec.rb
+- spec/lib/strategy_best_concurrency_spec.rb
+- spec/lib/tester_spec.rb
+- spec/spec_helper.rb
+- spec/tasks/spec.rake