kamal-railsbench 0.9.9.pre

data/lib/railsbench/benchmark_specs.rb

@@ -0,0 +1,63 @@
+require 'delegate'
+require 'yaml'
+require 'erb'
+
+class BenchmarkSpec < DelegateClass(Hash)
+  attr_accessor :name
+
+  READERS = %w(uri method post_data query_string new_session action controller session_data xhr raw_data)
+  READERS.each do |method|
+    define_method(method) { self[method] }
+  end
+
+  def initialize(name, hash)
+    super(hash)
+    @name = name
+  end
+
+  def inspect
+    "BenchmarkSpec(#{name},#{super})"
+  end
+
+  class << self
+    def load(name, file_name = nil)
+      unless file_name
+        file_name = ENV['RAILS_ROOT'] + "/config/benchmarks.yml"
+      end
+      @@specs = YAML::load(ERB.new(IO.read(file_name)).result)
+      raise "There is no benchmark named '#{name}'" unless @@specs[name]
+      parse(@@specs, name)
+    end
+
+    def parse(specs, name)
+      spec = specs[name]
+      if spec.is_a?(String)
+        spec.split(/, */).collect!{ |n| parse(specs, n) }.flatten
+      elsif spec.is_a?(Hash)
+        [ BenchmarkSpec.new(name,spec) ]
+      elsif spec.is_a?(Array)
+        spec.collect{|n| parse(specs, n)}.flatten
+      else
+        raise "oops: unknown entry type in benchmark specification"
+      end
+    end
+  end
+end
+
+__END__
+
+#  Copyright (C) 2007, 2008  Stefan Kaes
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA

data/lib/railsbench/gc_info.rb

@@ -0,0 +1,158 @@
+require File.expand_path(File.dirname(__FILE__) + "/perf_utils.rb")
+require "set"
+
+# Entry Format:
+#
+# Garbage collection started
+# objects processed: 0223696
+# live objects     : 0192126
+# freelist objects : 0000000
+# freed objects    : 0031570
+# kept 0000370 / freed 0000609 objects of type OBJECT
+# kept 0001071 / freed 0000062 objects of type CLASS
+# kept 0000243 / freed 0000061 objects of type ICLASS
+# kept 0000041 / freed 0000061 objects of type FLOAT
+# kept 0013974 / freed 0015432 objects of type STRING
+# kept 0000651 / freed 0000002 objects of type REGEXP
+# kept 0000617 / freed 0009948 objects of type ARRAY
+# kept 0000646 / freed 0001398 objects of type HASH
+# kept 0000004 / freed 0000121 objects of type BIGNUM
+# kept 0000006 / freed 0000005 objects of type FILE
+# kept 0000400 / freed 0000253 objects of type DATA
+# kept 0000001 / freed 0000093 objects of type MATCH
+# kept 0000067 / freed 0000136 objects of type VARMAP
+# kept 0000167 / freed 0000939 objects of type SCOPE
+# kept 0173634 / freed 0002389 objects of type NODE
+# GC time: 47 msec
+
+# Sentinel:
+# HEAP[ 0]: size= 300000
+# HEAP[ 1]: size= 600000
+# ...
+# number of requests processed: 1000
+# 128334 nodes malloced for 17345 KB
+# 396 leaks for 10464 total leaked bytes.
+
+
+GCAttributes = [:processed, :live, :freelist, :freed, :time]
+GCSummaries  = [:min, :max, :mean, :stddev, :stddev_percentage]
+
+class GCLogEntry
+  attr_accessor *GCAttributes
+  attr_accessor :live_objects, :freed_objects
+  def initialize
+    @live_objects = {}
+    @freed_objects = {}
+  end
+end
+
+class GCInfo
+
+  attr_reader(*GCAttributes)
+  attr_reader :entries, :num_requests, :collections, :garbage_produced, :time_total, :topology
+  attr_reader :live_objects, :freed_objects, :object_types, :garbage_totals
+  attr_reader :mallocs, :malloced, :leaks, :leaked
+
+  GCAttributes.each do |attr|
+    GCSummaries.each do |method|
+      attr_reader "#{attr}_#{method}"
+    end
+  end
+
+  def initialize(file)
+    @entries = []
+    @num_requests = 0
+    @topology = []
+    @object_types = Set.new
+    @mallocs = @malloced = @leaks = @leaked = 0
+
+    file.each_line do |line|
+      case line
+      when /^Garbage collection started$/
+        @entries << GCLogEntry.new
+      when /^objects processed\s*:\s*(\d+)$/
+        @entries.last.processed = $1.to_i
+      when /^live objects\s*:\s*(\d+)$/
+        @entries.last.live = $1.to_i
+      when /^freelist objects\s*:\s*(\d+)$/
+        @entries.last.freelist = $1.to_i
+      when /^freed objects\s*:\s*(\d+)$/
+        @entries.last.freed = $1.to_i
+      when /^GC time\s*:\s*(\d+)\s*msec$/
+        @entries.last.time = $1.to_i
+      when /^number of requests processed: (\d+)$/
+        @num_requests = $1.to_i
+      when /^HEAP\[\s*(\d+)\]: size=\s*(\d+)$/
+        @topology = [] if $1.to_i == 0
+        @topology << $2.to_i
+      when /^kept (\d+) \/ freed (\d+) objects of type ([a-zA-Z]+)/
+        @object_types.add($3)
+        @entries.last.live_objects[$3] = $1.to_i
+        @entries.last.freed_objects[$3] = $2.to_i
+      when /^(\d+) nodes malloced for (\d+) KB$/
+        @mallocs = $1.to_i
+        @malloced = $2.to_i * 1024
+      when /^(\d+) leaks for (\d+) total leaked bytes.$/
+        @leaks = $1.to_i
+        @leaked = $2.to_i
+      end
+    end
+
+    @time_total = @entries.map{|e| e.time}.sum
+    @collections = @entries.length
+    @garbage_produced = @entries.map{|e| e.freed}.sum
+    @live_objects = @entries.map{|e| e.live_objects}
+    @freed_objects = @entries.map{|e| e.freed_objects}
+    @freelist = @entries.map{|e| e.freelist}
+    @garbage_totals = @freed_objects.inject(Hash.new(0)) do |totals, freed|
+      freed.each do |object_type, count|
+        totals[object_type] += freed[object_type] || 0
+      end
+      totals
+    end
+
+    GCAttributes.each do |attr|
+      a = @entries.map{|e| e.send attr}
+      # we need to pop the last entry, as the freelist is not empty when doing the last forced GC
+      a.pop if :freelist == attr.to_sym
+
+      [:min, :max, :mean].each do |method|
+        instance_variable_set "@#{attr}_#{method}", (a.send method)
+      end
+      mean = instance_variable_get "@#{attr}_mean"
+      stddev = instance_variable_set "@#{attr}_stddev", (a.send :stddev, mean)
+      instance_variable_set "@#{attr}_stddev_percentage", stddev_percentage(stddev, mean)
+    end
+  end
+
+  OBJECT_TYPES = %w(NODE STRING ARRAY HASH SCOPE VARMAP CLASS ICLASS REGEXP FLOAT MATCH FILE DATA MODULE OBJECT)
+
+  class << self
+    def object_types(list_of_gc_infos)
+      list_of_gc_infos.inject(OBJECT_TYPES) do |object_types, gci|
+        (object_types + gci.object_types.to_a).uniq
+      end
+    end
+  end
+end
+
+
+### Local Variables: ***
+### mode:ruby ***
+### End: ***
+
+#    Copyright (C) 2006, 2007, 2008  Stefan Kaes
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

data/lib/railsbench/perf_info.rb

@@ -0,0 +1,146 @@
+require File.expand_path(File.dirname(__FILE__) + "/perf_utils.rb")
+
+# example of raw performance data
+
+# /home/skaes/railsbench/script/perf_bench 100 -bm=all -mysql_session -patched_gc -links -OT
+#                                       user     system      total        real
+# loading environment               0.954000   1.938000   2.892000 (  2.890000)
+# /empty/index                      0.093000   0.000000   0.093000 (  0.172000)
+# /welcome/index                    0.156000   0.000000   0.156000 (  0.172000)
+# /rezept/index                     0.125000   0.015000   0.140000 (  0.203000)
+# /rezept/myknzlpzl                 0.125000   0.000000   0.125000 (  0.203000)
+# /rezept/show/413                  0.406000   0.094000   0.500000 (  0.594000)
+# /rezept/cat/Hauptspeise           0.547000   0.094000   0.641000 (  0.688000)
+# /rezept/cat/Hauptspeise?page=5    0.531000   0.047000   0.578000 (  0.688000)
+# /rezept/letter/G                  0.422000   0.078000   0.500000 (  0.609000)
+# GC.collections=0, GC.time=0.0
+#                                       user     system      total        real
+# loading environment               0.813000   2.078000   2.891000 (  2.890000)
+# /empty/index                      0.125000   0.016000   0.141000 (  0.157000)
+# /welcome/index                    0.109000   0.000000   0.109000 (  0.187000)
+# /rezept/index                     0.110000   0.031000   0.141000 (  0.219000)
+# /rezept/myknzlpzl                 0.109000   0.016000   0.125000 (  0.219000)
+# /rezept/show/413                  0.422000   0.078000   0.500000 (  0.625000)
+# /rezept/cat/Hauptspeise           0.437000   0.125000   0.562000 (  0.656000)
+# /rezept/cat/Hauptspeise?page=5    0.453000   0.125000   0.578000 (  0.688000)
+# /rezept/letter/G                  0.438000   0.000000   0.438000 (  0.594000)
+# GC.collections=0, GC.time=0.0
+#                                       user     system      total        real
+# loading environment               0.938000   1.968000   2.906000 (  2.906000)
+# /empty/index                      0.109000   0.000000   0.109000 (  0.172000)
+# /welcome/index                    0.094000   0.031000   0.125000 (  0.171000)
+# /rezept/index                     0.110000   0.047000   0.157000 (  0.219000)
+# /rezept/myknzlpzl                 0.140000   0.016000   0.156000 (  0.203000)
+# /rezept/show/413                  0.422000   0.047000   0.469000 (  0.593000)
+# /rezept/cat/Hauptspeise           0.515000   0.015000   0.530000 (  0.672000)
+# /rezept/cat/Hauptspeise?page=5    0.484000   0.063000   0.547000 (  0.672000)
+# /rezept/letter/G                  0.453000   0.015000   0.468000 (  0.610000)
+# GC.collections=0, GC.time=0.0
+
+
+PerfAttributes = [:gc_calls, :gc_time, :load_time, :total_time]
+PerfSummaries  = [:min, :max, :mean, :stddev, :stddev_percentage]
+
+class PerfEntry
+  attr_accessor *PerfAttributes
+  attr_accessor :keys, :timings
+  def initialize
+    @keys = []
+    @timings = {}
+  end
+end
+
+class PerfInfo
+
+  attr_reader :options, :iterations, :keys
+  attr_reader :entries, :runs, :request_count, :requests_per_key
+
+  def gc_stats?
+    @gc_stats
+  end
+
+  PerfSummaries.each do |method|
+    PerfAttributes.each do |attr|
+      attr_reader "#{attr}_#{method}"
+    end
+    class_eval "def timings_#{method}(key); @timings[:#{method}][key]; end"
+  end
+
+  def initialize(file)
+    @entries = []
+    file.each_line do |line|
+      case line
+      when /^.*perf_([a-zA-Z.]+)\s+(\d+)\s+(.*)$/
+        @iterations = $2.to_i
+        @options = $3
+      when /\s+user\s+system\s+total\s+real/
+        @entries << PerfEntry.new
+      when /^(.*)\s+([\d\.]+)\s+([\d\.]+)\s+([\d\.]+)\s+\(\s*([\d\.]+)\s*\)$/
+        key, time = $1.strip, $5.to_f
+        if key == "loading environment"
+          @entries.last.load_time = time
+        else
+          @entries.last.keys << key
+          @entries.last.timings[key] = time
+        end
+      when /^GC.collections=(\d+), GC.time=([\d\.]+)$/
+        @entries.last.gc_calls, @entries.last.gc_time = [$1.to_i,$2.to_f]
+        @gc_stats = true
+      end
+    end
+
+    @entries.each{ |e| e.total_time =  e.timings.values.sum }
+    @keys = @entries.first.keys
+    @runs = @entries.length
+    if @keys.length == 1 && @keys[0] =~ /\((\d+) urls\)$/
+      @requests_per_key = $1.to_i
+    else
+      @requests_per_key = 1
+    end
+    @request_count = @iterations * @keys.length * @requests_per_key
+    @timings = PerfSummaries.inject({}){ |hash, method| hash[method] = Hash.new; hash }
+
+    @keys.each do |k|
+      a = @entries.map{|e| e.timings[k]}
+      [:min, :max, :mean].each do |method|
+        @timings[method][k] = a.send(method)
+      end
+      mean = @timings[:mean][k]
+      stddev = @timings[:stddev][k] = a.send(:stddev, mean)
+      @timings[:stddev_percentage][k] = stddev_percentage(stddev, mean)
+    end
+
+    PerfAttributes.each do |attr|
+      next if attr.to_s =~ /^gc_/ and !gc_stats?
+      a = @entries.map{|e| e.send attr}
+      [:min, :max, :mean].each do |method|
+        instance_variable_set "@#{attr}_#{method}", (a.send method)
+      end
+      mean = instance_variable_get "@#{attr}_mean"
+      stddev = instance_variable_set "@#{attr}_stddev", (a.send :stddev, mean)
+      instance_variable_set "@#{attr}_stddev_percentage", stddev_percentage(stddev, mean)
+    end
+
+  end
+end
+
+
+### Local Variables: ***
+### mode:ruby ***
+### End: ***
+
+#    Copyright (C) 2006, 2007, 2008  Stefan Kaes
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

data/lib/railsbench/perf_utils.rb

@@ -0,0 +1,202 @@
+# some utility methods
+
+class Array
+  def index_map
+    res = {}
+    each_with_index{|element, index| res[index] = element}
+    res
+  end
+
+  def restrict_to(index_set)
+    res = []
+    each_with_index{|e,i| res << e if index_set.include?(i)}
+    res
+  end
+
+  def sum
+    inject(0.0){|r,v| r += v }
+  end
+
+  def mean
+    sum/length
+  end
+
+  def stddev(mean=nil)
+    mean ||= self.mean
+    r = inject(0.0){|r,v| r += (v-mean)*(v-mean) }
+    Math.sqrt(r/(length-1))
+  end
+end
+
+def stddev_percentage(stddev, mean)
+  stddev.zero? ? 0.0 : (stddev/mean)*100
+end
+
+def determine_rails_root_or_die!(msg=nil)
+  unless ENV['RAILS_ROOT']
+    if File.directory?("config") && File.exists?("config/environment.rb")
+      ENV['RAILS_ROOT'] = File.expand_path(".")
+    else
+      die(msg || "#{File.basename $PROGRAM_NAME}: $RAILS_ROOT not set and could not be configured automatically")
+    end
+  end
+end
+
+def die(msg, error_code=1)
+  $stderr.puts msg
+  exit error_code
+end
+
+class File
+  def self.open_or_die(filename, &block)
+    filename = filename.sub(/^\/([cdefgh])(\/)/, '\1:\2') if RUBY_PLATFORM =~ /win32/
+    begin
+      if stat(filename).readable?
+        open(filename, &block)
+      else
+        die "file #{filename} is unreadable"
+      end
+    rescue
+      die "file #{filename} does not exist"
+    end
+  end
+end
+
+def truncate(text, length = 32, truncate_string = "...")
+  if text.nil? then return "" end
+  l = truncate_string.length + 1
+
+  if RUBY_VERSION !~ /1.9/ && $KCODE == "NONE"
+    text.length > length ? text[0..(length - l)] + truncate_string : text
+  else
+    chars = text.split(//)
+    chars.length > length ? chars[0..(length - l)].join + truncate_string : text
+  end
+end
+
+RAILSBENCH_BINDIR = File.expand_path(File.dirname(__FILE__) + "/../../script")
+
+def enable_gc_stats(file)
+  ENV['RUBY_GC_STATS'] = "1"
+  ENV['RUBY_GC_DATA_FILE'] = file
+end
+
+def disable_gc_stats
+  ENV.delete 'RUBY_GC_STATS'
+  ENV.delete 'RUBY_GC_DATA_FILE'
+end
+
+def unset_gc_variables
+  %w(RUBY_HEAP_MIN_SLOTS RUBY_GC_MALLOC_LIMIT RUBY_HEAP_FREE_MIN).each{|v| ENV.delete v}
+end
+
+def load_gc_variables(gc_spec)
+  File.open_or_die("#{ENV['RAILS_ROOT']}/config/#{gc_spec}.gc").each_line do |line|
+    ENV[$1] = $2 if line =~ /^(?:export )?(.*)=(.*)$/
+  end
+end
+
+def set_gc_variables(argv)
+  gc_spec = nil
+  argv.each{|arg| gc_spec=$1 if arg =~ /-gc=([^ ]*)/}
+
+  if gc_spec
+    load_gc_variables(gc_spec)
+  else
+    unset_gc_variables
+  end
+end
+
+def benchmark_file_name(benchmark, config_name, prefix=nil, suffix=nil)
+  perf_data_dir = (ENV['RAILS_PERF_DATA'] ||= ENV['HOME'])
+  date = Time.now.strftime '%m-%d'
+  suffix = ".#{suffix}" if suffix
+  ENV['RAILS_BENCHMARK_FILE'] =
+    if config_name
+      "#{perf_data_dir}/#{date}.#{benchmark}.#{config_name}#{suffix}.txt"
+    else
+      "#{perf_data_dir}/perf_run#{prefix}.#{benchmark}#{suffix}.txt"
+    end
+end
+
+def quote_arguments(argv)
+  argv.map{|a| a.include?(' ') ? "'#{a}'" : a.to_s}.join(' ')
+end
+
+def ruby
+  ENV['RUBY'] || "ruby"
+end
+
+def perf_run(script, iterations, options, raw_data_file)
+  perf_runs = (ENV['RAILS_PERF_RUNS'] ||= "3").to_i
+
+  disable_gc_stats
+  set_gc_variables([iterations, options])
+
+  perf_options = "#{iterations} #{options}"
+  null = (RUBY_PLATFORM =~ /win32/i) ? 'nul' : '/dev/null'
+
+  perf_cmd = "#{ruby} #{RAILSBENCH_BINDIR}/perf_bench #{perf_options}"
+  print_cmd = "#{ruby} #{RAILSBENCH_BINDIR}/perf_times #{raw_data_file}"
+
+  puts "benchmarking #{perf_runs} runs with options #{perf_options}"
+
+  File.open(raw_data_file, "w"){ |f| f.puts perf_cmd }
+  perf_runs.times do
+    system("#{perf_cmd} >#{null}") || die("#{script}: #{perf_cmd} returned #{$?}")
+  end
+  File.open(raw_data_file, "a" ){|f| f.puts }
+
+  unset_gc_variables
+  system(print_cmd) || die("#{script}: #{print_cmd} returned #{$?}")
+end
+
+def perf_run_gc(script, iterations, options, raw_data_file)
+  warmup = "-warmup"
+  warmup = "" if options =~ /-warmup/
+
+  enable_gc_stats(raw_data_file)
+  set_gc_variables([options])
+
+  perf_options = "#{iterations} #{warmup} #{options}"
+  null = (RUBY_PLATFORM =~ /win32/) ? 'nul' : '/dev/null'
+
+  perf_cmd = "#{ruby} #{RAILSBENCH_BINDIR}/run_urls #{perf_options} >#{null}"
+  print_cmd = "#{ruby} #{RAILSBENCH_BINDIR}/perf_times_gc #{raw_data_file}"
+
+  if options =~ /-leaks/
+    if RUBY_PLATFORM =~ /darwin9/
+      puts "enabling MallocStackLogging"
+      perf_cmd.insert(0, "MallocStackLogging=1 ")
+    else
+      die "leak debugging not supported on #{RUBY_PLATFORM}"
+    end
+  end
+
+  puts "benchmarking GC performance with options #{perf_options}"
+  puts
+
+  system(perf_cmd) || die("#{script}: #{perf_cmd} returned #{$?}")
+
+  disable_gc_stats
+  unset_gc_variables
+  system(print_cmd) || die("#{script}: #{print_cmd} returned #{$?}")
+end
+
+__END__
+
+#  Copyright (C) 2005-2008  Stefan Kaes
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA