herokubench 0.0.9 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cb2da4a4361fae8687d6f1000bc956fa668f09ab
-  data.tar.gz: 1468528eb35ebf73b7cb01cf16b0a19f9c7017a1
+  metadata.gz: 1991f6c51c9f3abcdd15975c79aaa8cad9ab970a
+  data.tar.gz: 125fa53e62a2ca9189dab4af4a64fcfd1ec6f3db
 SHA512:
-  metadata.gz: fbad4b333c5c4e3704634f4cef8481b16928d92efd1f10ffd62b35aec857b581fa654cd4e78c1851555378e170e6f542966bb80593e9bd04b64fe35190e6822e
-  data.tar.gz: 40dc3ab2b9b76868efacbd43ac1f7663fd4467c446ef91fb2cef612e02472d593eb56bdaa0a4ce44add5d86f12c5e95e9eaaff7565504169cfeb49a0ab2ec903
+  metadata.gz: 23b53138b1c1ec6ed1926ebb01194b41956ff1aeaab2c10b92cd4a0cc207d9becef7f7ebd6dc39f0222d7c677f663052d271ffc21a84f34b68d1450362579e3e
+  data.tar.gz: 76e4c1fae840664be549815e7ae54064bf77ef21e65d8c0d03cfc882e79682da55e26be75238131ac66d8e30dcde8fc9a1bf7a336418a62737d13548eab327c5

data/README.md

@@ -8,29 +8,29 @@ A build server in the cloud, heavily inspired by Vulcan.
 
 ## Usage
 
-    $ hb help
-    ommands:
-      hb ab URL           # Run apache-bench, using a single, one-off Heroku dyno
-      hb create APP_NAME  # Create your personal bench-server on Heroku
-      hb help [COMMAND]   # Describe available commands or one specific command
-      hb multi URL        # Run apache-bench, using multiple one-off dynos
-    
+    $ hbench help
+    commands:
+      hbench ab [options] [http[s]://]hostname[:port]/path            # Run apache-bench using a single one-off dyno
+      hbench create APP_NAME                                          # Create your personal bench-server on Heroku
+      hbench help [COMMAND]                                           # Describe available commands or one specific command
+      hbench multi NUMBER [options] [http[s]://]hostname[:port]/path  # Run apache-bench, using multiple one-off dynos
+      hbench update                                                   # Updates your remote bench server
+
     Options:
       [--verbose]
 
 
-
 ## Examples
 
 ### Create a Bench Server
-    $ hb create hbench-david
+    $ hbench create hbench-david
     Creating hbench-david... done, stack is cedar
     http://hbench-david.herokuapp.com/ | git@heroku.com:hbench-david.git
     ...
 
 ### Bench
 
-    $ hb http://nodejssimple.herokuapp.com/
+    $ hbench -c 100 -n 1000 http://nodejssimple.herokuapp.com/
       Running one-off dyno, please be patient
       Running `ab -c 1000 -n 10000 http://nodejssimple.herokuapp.com/` attached to terminal... up, run.4045
       This is ApacheBench, Version 2.3 <$Revision: 1430300 $>

data/lib/herokubench/cli.rb

@@ -21,10 +21,9 @@ require 'herokubench/result'
 # This class is based upon Vulcan, and copies heavily.
 
 class HerokuBench::CLI < Thor
-  class_option "verbose",  :type => :boolean
+  class_option :verbose, :type => :boolean
   check_unknown_options!  :except => [:ab, :multi]
   default_task :ab
-  class_options["verbose"] = false if class_options["verbose"].nil?
   Heroku.user_agent = "heroku-gem/#{Heroku::VERSION} (#{RUBY_PLATFORM}) ruby/#{RUBY_VERSION}"
   Heroku::Command.load
 
@@ -45,87 +44,87 @@ class HerokuBench::CLI < Thor
     update
   end
 
-
-  desc "ab [options] [http[s]://]hostname[:port]/path", "Run apache-bench using a single one-off dyno"
+  desc "ab [options] [http[s]://]hostname[:port]/path", "Run apache-bench automatically spreading over dynos as necessary"
   long_desc <<-LONGDESC
-  'hb ab' will run apache-bench, using a one-off dyno on Heroku in order
-  to best benchmark the performance of your webservice.
+  'hb ab' will run apache-bench, splitting the work between up between as many dynos as necessary, for a maximum of
+  100 concurrent connections per dyno. The arguments are passed directly to ab.
 
   For more information, run `hb ab help`
 
-  > $ hb ab -c 100 -n 1000 http://www.google.com
+  > $ hbench ab -c 100 -n 1000 http://www.google.com
   LONGDESC
 
   def ab(*args)
-    puts config[:app]
-    error "no app yet, please create first" unless config[:app]
-    puts "Running one-off dyno, please be patient"
-    Heroku::Command.run("run", ["ab #{args.join(' ')}", "--app", "#{config[:app]}"])
+      num_requests_index = args.index("-n") #hack to extract number of requests from the ab arguments.
+      concurrency_level_index = args.index("-c") #hack to extract number of requests from the ab arguments.
+      unless concurrency_level_index.nil? or num_requests_index.nil? then
+        num_requests = args[num_requests_index + 1].to_i
+        concurrency_level = args[concurrency_level_index + 1].to_i
+
+        num_dynos = (concurrency_level/100.0).ceil
+        num_dynos = 1 if num_dynos ==0
+
+        say "Inferred #{num_dynos} instances" if options[:verbose]
+        args[args.index("-n") + 1] = (num_requests / num_dynos).to_i
+        args[args.index("-c") + 1] = (concurrency_level / num_dynos).to_i
+      end
+      num_dynos ||= 1
+      multi(num_dynos, *args)
   end
 
-
-  desc "multi NUMBER [options] [http[s]://]hostname[:port]/path", "Run apache-bench, using multiple one-off dynos"
+  desc "multi NUMDYNOS [options] [http[s]://]hostname[:port]/path", "Run apache-bench, using multiple one-off dynos"
   long_desc <<-LONGDESC
-  'hb multi' will run apache-bench, using multiple one-off dynos in order
+  'hbench multi' will run apache-bench, using a specfied number of dynos in order
   to incrase the throughput of your benchmark.
 
   The arguments are identical to that of 'hb ab' with the addition
-  of the 'NUMBER' argument, representing the number of one-off dynos
+  of the 'NUMDYNOS' argument, representing the number of one-off dynos
   to execute your benchmark on.
 
-  > $ hb multi 5 http://www.google.com
+  > $ hbench multi 5 http://www.google.com
   LONGDESC
   def multi(dynos, *args)
     error "no app yet, create first" unless config[:app]
     error "Number of dynos must be an integer greater than 1" unless dynos.to_i >= 1
+
     begin
-      num_requests = args[args.index("-n") + 1].to_i #hack to extract number of requests from the ab arguments.
+      say "Using #{config[:app]}" if options[:verbose] 
+      say "Benching with #{dynos} dynos and arguments #{args}" if options[:verbose]
 
       bencher_path = File.expand_path("../bencher.rb",__FILE__)
+      num_dynos = dynos.to_i
 
-      numdynos = dynos.to_i
-      results = []
       running_procs = {}
-      progesses = {}
       ab_command = "ab #{args.join(' ')}"
 
-      p_bar = ProgressBar.create(:title=>'Benching',:total=>numdynos*num_requests, :smoothing => 0.6)
+      p_bar = ProgressBar.create(:title=>'Benching',:total=>1 + num_dynos*100)
+      summary_result  = ApacheBenchSummaryResult.new
 
-      numdynos.times do  |n|
+      num_dynos.times do  |n|
         t_file = Tempfile.new("hbench_out_#{n}") 
         pid = spawn( "ruby #{bencher_path} \"#{ab_command} \" --app #{config[:app]}", :out=>t_file.path, :err=>null_dev)
+
         running_procs[pid] = t_file
-        progesses[pid] = 0
-        puts t_file.path
+        summary_result.add_result(ApacheBenchResult.new(t_file))
+        puts t_file.path if options[:verbose]
       end
 
 
       until running_procs.empty?
         begin
-          complete_results = Timeout.timeout(1) do
+          complete_results = Timeout.timeout(0.5) do
             pid = Process.wait
-            results.push running_procs.delete(pid)
-            p_bar.progress += num_requests - progesses[pid]
-            progesses[pid] = num_requests
+            running_procs.delete(pid)
           end
         rescue Timeout::Error
-          running_procs.each do |pid, tfile|
-            progress = get_progress(tfile)
-            p_bar.progress += progress - progesses[pid]
-            progesses[pid] = progress
-          end
+          diff = summary_result.get_progress() - p_bar.progress
+          p_bar.progress+= diff
         end
       end
-      
-      summary_result  = ApacheBenchSummaryResult.new
 
-      puts results
-      results.each  do |tfile|
-        summary_result.add_result(ApacheBenchResult.new(tfile))
-      end 
+      p_bar.finish
       summary_result.print()
 
-
     rescue Interrupt
       say "Exiting...Please be patient"
       kill_running_procs(running_procs)
@@ -133,71 +132,37 @@ class HerokuBench::CLI < Thor
       say "Done"
     rescue => exception
       say("HerokuBench ran into an unexpected exception. Please contact @wcdolphin",:red)
+
+      begin
+        kill_running_procs(running_procs)
+        kill_running_dynos()
+      rescue
+      end #squelch exceptions when killing procs
+
       say(exception,:red)
       puts exception.backtrace
     end
   end
 
-  private
 
-  # Attemptes to parse a value as a Float or integer, defaulting to the original
-  # string if unsuccesful.
-  def parse(v)
-    ((float = Float(v)) && (float % 1.0 == 0) ? float.to_i : float) rescue v
-  end
 
-  # pretty much the opposite of parse. Returns a string of the best way
-  # to represent a float, int or string value
-  def serialize(v)
-    ((float = v.round(1)) && (float % 1.0 == 0) ? float.to_i.to_s : float.to_s) rescue v.to_s
-  end
-
-  def get_progress(tfile)
-    tfile.rewind
+  desc "update", "Updates your remote bench server"
+  def update
+    error "No app yet, create first" unless config[:app]
 
-    progress = tfile.each_line.collect do |line|
-      group = line.scan(/Completed (\d+) requests/)
-      group = group.empty? ? 0 : group[0][0]
+    Dir.mktmpdir do |dir|
+      Dir.chdir(dir) do
+        system "git init -q"
+        system "git remote add origin git@github.com:wcdolphin/heroku-benchserver.git"
+        system "git remote add heroku git@#{heroku_git_domain}:#{config[:app]}.git"
+        pullres = capture { system "git pull --quiet origin master"}
+        pushres = capture { system "git push heroku master --quiet"}
+      end
     end
-    progress = progress.last.to_i
   end
-  # def summarize(results)
-  #   summary = {}
-
-  #   results.each do |result|
-  #     result.each do |type, hash|
-  #       summary[type] = {} if summary[type].nil?
-  #       hash.each do |k,v|
-  #         summary[type][k] = [0.0] * v.length if summary[type][k].nil?
-  #         v.each_index do |i|
-  #           if not @@summable_fields.index(k).nil?
-  #             summary[type][k][i] += v[i]
-  #           elsif not @@medianable_fields.index(k).nil?
-  #             summary[type][k][i] += v[i] / results.length.to_f
-  #           elsif not @@maxable_fields.index(k).nil?
-  #             summary[type][k][i] = [v[i], summary[type][k][i]].max
-  #           else
-  #             summary[type][k][i] = v[i]
-  #           end
-  #         end
-  #       end
-  #     end
-  #   end
-
-  #   say "\tCumulative results, summed across dynos"
-  #   say ""
-  #   summary[:generic_result].each{|k,v| say format(k+":",v, 25)}
-
-  #   say ""
-  #   say "\t Connection Times (ms), median across dynos"
-  #   say format("",["min", "mean", "[+/-sd]" ,"median","max"],15)
-  #   summary[:connection_times].each{|k,v| say format(k+":",v, 15)}
-
-  #   say ""
-  #   say "\t Percentage of the requests served within a certain time (ms)"
-  #   say "\t across dynos"
-  #   summary[:response_time_cdf].each{|k,v| say format(k+"",v, 15)}
-  # end
+
+  private
+
 
   def kill_running_procs(running_procs)
       print "Killing running processes"
@@ -219,45 +184,6 @@ class HerokuBench::CLI < Thor
     end
   end
 
-  def format(k,v, pad)
-    "#{fill(k,pad)}#{v.map{|val| fill(serialize val)}.join('')}\r\n"
-  end
-
-  def fill(str, length=12)
-    "#{str}#{" " * (length - str.length)}"
-  end
-
-  def get_result_hash(f)
-    result_hash = {}
-    f.each_line do |line|
-      @@result_type.each do |k,v|
-        group = line.scan(v)
-        if not group.nil? and group.length.equal? 1
-          capture = group[0].map {|v| parse v} #convert to float/int/etc
-          result_hash[k] = {} unless result_hash.has_key?(k)
-          res_key = capture[0]
-          res_values = capture.slice(1, capture.length)
-          result_hash[k][res_key] = res_values
-        end
-      end
-    end
-    result_hash
-  end
-
-  def update
-    error "no app yet, create first" unless config[:app]
-
-    Dir.mktmpdir do |dir|
-      Dir.chdir(dir) do
-        system "git init -q"
-        system "git remote add origin git@github.com:wcdolphin/heroku-benchserver.git"
-        system "git remote add heroku git@#{heroku_git_domain}:#{config[:app]}.git"
-        pullres = capture { system "git pull --quiet origin master"}
-        pushres = capture { system "git push heroku master --quiet"}
-      end
-    end
-  end
-
   def capture
     results = $stdout = StringIO.new
     yield

data/lib/herokubench/result.rb

@@ -19,110 +19,147 @@ require 'tempfile'
 require 'ruby-progressbar'
 
 
-class BaseResult
+class BaseResult < Thor
 	@@summable_fields = []
 	@@maxable_fields = []
 end
 
 class ApacheBenchResult < BaseResult
-  attr_accessor :result_hash
-  @@result_regexes = {
-    :connection_times=>/(.+):\s+\s+([\d|\.]+)\s+([\d|\.]+)\s+([\d|\.]+)\s+([\d|\.]+)\s+([\d|\.]+)/,
-    :generic_result=> /^([\w\s]+):\s*([\d|\.]+)/,
-    :response_time_cdf=>/(\d+%)\s+(\d+)/
-  }
-  @@summable_fields = ["Complete requests","Failed requests", "Write errors", "Requests per second", "Total transferred", "HTML transferred", "Concurrency Level"]
-  @@averageable_fields = ["Connect", "Processing", "Waiting", "Total", "Time per request", "Time taken for tests"]
-  @@maxable_fields    = ["50%", "66%", "75%", "80%", "90%", "95%", "98%", "99%","100%"]
-
-	def initialize(temp_file)
-		@result_hash = {}
-
-		temp_file.each_line do |line|
-		  @@result_regexes.each do |type,v|
-		    group = line.scan(v)
-		    if not group.nil? and group.length.equal? 1
-		      capture = group[0].map {|v| parse v} #convert to float/int/etc
-		      res_key = capture[0]
-		      res_values = capture.slice(1, capture.length)
-
-		      @result_hash[type] = {} unless @result_hash.has_key? type
-		      @result_hash[type][res_key] = res_values.length == 1 ? res_values[0] : res_values  
-		    end
-		  end
-		 end
-		puts @result_hash
-	end
 
+    attr_accessor :result_tfile
+    @@result_regexes = {
+      :connection_times=>/(.+):\s+\s+([\d|\.]+)\s+([\d|\.]+)\s+([\d|\.]+)\s+([\d|\.]+)\s+([\d|\.]+)/,
+      :generic_result=> /^([\w\s]+):\s*([\d|\.]+)/,
+      :response_time_cdf=>/(\d+%)\s+(\d+)/
+    }
+    @@summable_fields = ["Complete requests","Failed requests", "Write errors", "Requests per second", "Total transferred", "HTML transferred", "Concurrency Level"]
+    @@averageable_fields = ["Connect", "Processing", "Waiting", "Total", "Time per request", "Time taken for tests"]
+    @@maxable_fields    = ["50%", "66%", "75%", "80%", "90%", "95%", "98%", "99%","100%"]
+
+  no_commands do
+  	def initialize(temp_file)
+      @result_tfile = temp_file
+  	end
+
+  	def result_hash()
+      @result_tfile.rewind
+      resulting_hash = {}
+      @result_tfile.each_line do |line|
+        @@result_regexes.each do |type,v|
+          group = line.scan(v)
+          if not group.nil? and group.length.equal? 1
+            capture = group[0].map {|v| parse v} #convert to float/int/etc
+            res_key = capture[0]
+            res_values = capture.slice(1, capture.length)
+
+            resulting_hash[type] = {} unless resulting_hash.has_key? type
+            resulting_hash[type][res_key] = res_values.length == 1 ? res_values.first : res_values  
+            break
+          end
+        end
+       end
+       resulting_hash
+  	end
+
+
+    # Returns the progress of this particular Dyno/Bench Result
+    def get_progress()
+      @result_tfile.rewind
+      progress = @result_tfile.each_line.collect do |line|
+        group = line.scan(/Completed (\d+) requests/)
+        group = group.empty? ? 0 : group[0][0]
+      end
+      progress.reject{|x| x == 0}.length * 10
+    end
+
+
+    def unlink()
+      @result_tfile.unlink
+    end
+  end
 end
 
 class ApacheBenchSummaryResult < ApacheBenchResult
+	no_commands do
+		def initialize()
+			@results = []
+		end
 
-	def initialize()
-		@results = []
-	end
-
-	def add_result(ab_result)
-		@results.push ab_result
-	end
+		def add_result(ab_result)
+			@results.push ab_result
+		end
 
-	def get_summary_result()
-		summary_result_hash = {}
+    def get_progress()
+      @results.collect{|r| r.get_progress}.inject(:+)
+    end
+
+    def unlink()
+      @results.each{|r| r.unlink}
+    end
+
+		def get_summary_result()
+			summary_result_hash = {}
+			@results.each do |result|
+				result.result_hash.each do |result_type, res_type_hashes| 
+					summary_result_hash[result_type] = {} unless summary_result_hash.has_key? result_type
+					res_type_hashes.each do |result_key, value|
+						summary_result_hash[result_type][result_key] = [] unless summary_result_hash[result_type].has_key? result_key
+						summary_result_hash[result_type][result_key].push value
+					end
+				end	
+			end
 
-		@results.each do |result|
-			result.result_hash.each do |result_type, res_type_hashes| 
-				summary_result_hash[result_type] = {} unless summary_result_hash.has_key? result_type
-				res_type_hashes.each do |result_key, value|
-					summary_result_hash[result_type][result_key] = [] unless summary_result_hash[result_type].has_key? result_key
-					summary_result_hash[result_type][result_key].push value
+			summary_result_hash.each do |result_type, result_hash|
+				result_hash.each do |result_name, result_values| 
+					if @@summable_fields.include? result_name
+						summary_result_hash[result_type][result_name] = result_values.inject(:+)			
+					elsif @@averageable_fields.include? result_name
+						summary_result_hash[result_type][result_name] = deep_average(result_values)
+					elsif  @@maxable_fields.include? result_name
+						summary_result_hash[result_type][result_name] = result_values.max
+					else
+						summary_result_hash[result_type][result_name] = result_values.first
+					end
 				end
-			end	
+			end
+
+			summary_result_hash  
 		end
 
-		summary_result_hash.each do |result_type, result_hash|
-			result_hash.each do |result_name, result_values| 
-				if @@summable_fields.include? result_name
-					summary_result_hash[result_type][result_name] = result_values.inject(:+)			
-				elsif @@averageable_fields.include? result_name
-					summary_result_hash[result_type][result_name] = deep_average(result_values)
-				elsif  @@maxable_fields.include? result_name
-					summary_result_hash[result_type][result_name] = result_values.max
-				else
-					summary_result_hash[result_type][result_name] = result_values.first
+		def print
+			summary = self.get_summary_result()
+			if summary.empty?
+				say("Herokubench ran into an error while executing ApacheBench. It is likely there was a syntax error in your command. Please see output below.", :red)
+				@results.first.result_tfile.rewind
+				@results.first.result_tfile.each_line do |line|
+					puts line
 				end
+				return
+			end
+			say("Cumulative results, summed across dynos",:bold) 
+			summary[:generic_result].each{|k,v| printf "%-20s %s\n", k + ":",v}
+
+      say ""
+      say("Connection Times (ms), median across dynos",:bold)
+			printf "%-20s %-8s %-8s %-8s %s\n", "","min", "mean", "[+/-sd]" ,"median","max"
+			summary[:connection_times].each do |k,v|
+				printf "%-20s %-8s %-8s %-8s %s\n",k +":", v[0], v[1], v[2], v[3], v[4], v[5]
 			end
-		end
 
-		summary_result_hash		
-	end
-	def print
-		# puts "-----"
-		summary = self.get_summary_result()
-		# puts summary
-		puts "-----"
-		puts "\t Cumulative results, summed across dynos"
-		puts ""
-		summary[:generic_result].each{|k,v| printf "%-20s %s\n", k + ":",v}
-
-		puts ""
-		puts "\t Connection Times (ms), median across dynos"
-		printf "%-20s %-6s %-6s %-6s %s\n", "","min", "mean", "[+/-sd]" ,"median","max"
-		summary[:connection_times].each do |k,v|
-			printf "%-20s %-6s %-6s %-6s %s\n",k +":", v[0], v[1], v[2], v[3], v[4], v[5]
+      if summary.has_key? :response_time_cdf #valid ab results can have no result CDF if there is only one response
+        say ""
+        say("Percentage of the requests served within a certain time (ms) across dynos", :bold)
+        summary[:response_time_cdf].each{|k,v| printf "\t%-20s %s\n", k,v}
+      end
 		end
-		puts ""
-		puts "\t Percentage of the requests served within a certain time (ms)"
-		puts "\t across dynos"
-		summary[:response_time_cdf].each{|k,v| printf "  %-20s %s\n", k,v}
-
 	end
-
 end
 
 
 
 private
-
+# Parses a value as a Float or integer, defaulting to the original
+# string if unsuccesful.
 def parse(v)
   ((float = Float(v)) && (float % 1.0 == 0) ? float.to_i : float) rescue v
 end
@@ -131,13 +168,10 @@ def deep_average(arr)
 	result = []
 	if not arr.empty? and arr[0].is_a? Array #we need to do a 'deep' average.
 		arr[0].each_index do |i|
-			result[i] = arr.collect {|a| a[i]/arr.length.to_f}.inject(:+)
+			result[i] = arr.collect {|a| a[i]/arr.length.to_f}.inject(:+).round(1)
 		end
 		result
 	else	
-		arr.collect {|a| a/arr.length}.inject(:+)
+		arr.collect {|a| a/arr.length}.inject(:+).round(1)
 	end
 end
-
-
-

data/lib/herokubench/version.rb

@@ -1,5 +1,5 @@
 module HerokuBench
 
-  VERSION = "0.0.9"
+  VERSION = "1.0.0"
 
 end

data/spec/cli_spec.rb

@@ -0,0 +1,7 @@
+$:.unshift File.expand_path("../lib", __FILE__)
+
+require 'herokubench/result'
+
+describe HerokuBench::CLI do
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: herokubench
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 1.0.0
 platform: ruby
 authors:
 - Cory Dolphin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-11 00:00:00.000000000 Z
+date: 2013-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: heroku
@@ -82,6 +82,7 @@ files:
 - lib/herokubench/version.rb
 - lib/herokubench.rb
 - README.md
+- spec/cli_spec.rb
 homepage: https://github.com/wcdolphin/heroku-bench
 licenses:
 - MIT
@@ -102,7 +103,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.0.6
 signing_key: 
 specification_version: 4
 summary: A gem to help load testing web applications deployed on AWS or Heroku, using