railsbench 0.8.4

data/script/perf_bench

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+unless ENV['RAILS_ROOT']
+  $stderr.puts("\nOOPS: can't benchmark unless RAILS_ROOT is set")
+  $stderr.puts("\nbenchmarking aborted!")
+  exit 1
+end
+
+trap("INT") { $stderr.puts("benchmarking aborted!"); exit!(-1) }
+
+module Benchmark
+  if ENV['RAILS_BENCHMARK_FILE']
+    OUTPUT = File.open(ENV['RAILS_BENCHMARK_FILE'], "a+")
+    SYNC = false
+  else
+    OUTPUT = STDERR
+  end
+end
+require "benchmark"
+include Benchmark
+
+bm(32) do |test|
+  test.report("loading environment") do
+    require 'railsbench/railsbenchmark'
+    require ENV['RAILS_ROOT'] + '/config/benchmarks'
+  end
+
+  trap("INT") do
+    $stderr.puts "clearing database connections ..."
+    ActiveRecord::Base.send :clear_all_cached_connections!
+    $stderr.puts "benchmarking aborted!"
+    exit!(-1)
+  end
+
+  benchmarks = YAML::load(File.open(ENV['RAILS_ROOT'] + "/config/benchmarks.yml"))
+  benchmark_name = "default"
+  ARGV.each{ |arg| benchmark_name = $1 if arg =~ /-bm=([a-zA-Z_0-9]+)/ }
+ 
+  RAILS_BENCHMARKER.iterations = ARGV[0].to_i
+  RAILS_BENCHMARKER.url_spec = benchmarks
+  RAILS_BENCHMARKER.setup_test_urls(benchmark_name)
+  RAILS_BENCHMARKER.establish_test_session
+  RAILS_BENCHMARKER.warmup
+  
+  if ARGV.include?('-mix')
+    RAILS_BENCHMARKER.run_url_mix(test)
+  else
+    RAILS_BENCHMARKER.run_urls(test)
+  end
+end
+
+Benchmark::OUTPUT.close unless Benchmark::OUTPUT.nil? || !ENV['RAILS_BENCHMARK_FILE']
+
+__END__
+
+### Local Variables: ***
+### mode:ruby ***
+### End: ***
+
+#  Copyright (C) 2005, 2006  Stefan Kaes
+# 
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+# 
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA

data/script/perf_comp

@@ -0,0 +1,155 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+require 'railsbench/perf_info'
+
+narrow = ARGV.delete('-narrow') || ARGV.delete('-n')
+skip_urls = ARGV.delete('-skip_urls') || ARGV.delete('-s')
+
+if ARGV.length < 2
+  $stderr.puts "perfcomp: not enough arguments!"
+  $stderr.puts "usage: perf_comp [-n[arrow]] [-s[kip_urls]] file1 file2"
+  exit 1
+elsif ARGV.length > 2
+  $stderr.puts "perfcomp: too many arguments!"
+  $stderr.puts "usage: perf_comp [-n[arrow]] [-s[kip_urls]] file1 file2"
+  exit 1
+end
+
+files = []
+configs = []
+pi = []
+
+ARGV.each do |arg|
+  files << File.open_or_die(arg)
+  pi << PerfInfo.new(files.last)
+end
+
+if pi[0].keys != pi[1].keys
+  $stderr.puts "perfcomp: can't compare runs with differing requests!"
+  exit 1
+else
+  keys = pi[0].keys
+end
+
+iter1 = pi[0].iterations
+iter2 = pi[1].iterations
+
+if iter1 != iter2
+  $stderr.puts "perfcomp: scaling performance data: iterations of run 1 and run 2 differ: #{iter1}/#{iter2}"
+end
+
+if iter1 < iter2
+  c1 = iter2 / iter1
+  c2 = 1
+else
+  c1 = 1
+  c2 = iter1 / iter2
+end
+
+printf "perf data file 1: #{files[0].path}\n"
+printf "  requests=#{iter1}, options=#{pi[0].options}\n\n"
+printf "perf data file 2: #{files[1].path}\n"
+printf "  requests=#{iter2}, options=#{pi[1].options}\n\n"
+
+if narrow
+  printf "%-3s %9s %9s %7s %7s %8s %8s %6s\n",
+         'page', 'c1 real', 'c2 real', 'c1 r/s', 'c2 r/s', 'c1 ms/r', 'c2 ms/r', 'c1/c2'
+else
+  printf "%-32s %9s %9s %7s %7s %8s %8s %6s\n",
+         'page', 'c1 real', 'c2 real', 'c1 r/s', 'c2 r/s', 'c1 ms/r', 'c2 ms/r', 'c1/c2'
+end
+
+keys.each_with_index do |k, index|
+  # average runtime
+  t1 = pi[0].timings_mean(k) * c1
+  t2 = pi[1].timings_mean(k) * c2
+  urls1 = pi[0].requests_per_key
+  urls2 = pi[1].requests_per_key
+
+  # requests per second
+  rps1 = iter1*urls1*c1/t1
+  rps2 = iter2*urls2*c2/t2
+
+  # milliseconds per request
+  mspr1 = t1*1000/(iter1*urls1*c1)
+  mspr2 = t2*1000/(iter2*urls2*c2)
+
+  if narrow
+    printf "%2d:  %9.5f %9.5f %7.1f %7.1f %8.2f %8.2f %6.2f\n",
+           (index+1), t1, t2, rps1, rps2, mspr1, mspr2, t1/t2
+  else
+    printf "%-32s %9.5f %9.5f %7.1f %7.1f %8.2f %8.2f %6.2f\n",
+           truncate(k), t1, t2, rps1, rps2, mspr1, mspr2, t1/t2
+  end
+end
+
+total_requests_run1 = pi[0].request_count
+total_requests_run2 = pi[1].request_count
+total_time1 = pi[0].total_time_mean * c1
+total_time2 = pi[1].total_time_mean * c2
+total_rps1 = total_requests_run1 * c1 / total_time1
+total_rps2 = total_requests_run2 * c2 / total_time2
+total_mspr1 = total_time1 * 1000 / (total_requests_run1 * c1)
+total_mspr2 = total_time2 * 1000 / (total_requests_run2 * c2)
+
+if narrow
+  printf "\n%4s %9.5f %9.5f %7.1f %7.1f %8.2f %8.2f %6.2f\n",
+           "all:", total_time1, total_time2,
+                total_rps1, total_rps2, total_mspr1, total_mspr2,
+                     total_time1/total_time2
+else
+  printf "\n%-32s %9.5f %9.5f %7.1f %7.1f %8.2f %8.2f %6.2f\n",
+           "all requests", total_time1, total_time2,
+                total_rps1, total_rps2, total_mspr1, total_mspr2,
+                     total_time1/total_time2
+end
+
+if pi[0].gc_stats? && pi[1].gc_stats?
+  c1 = pi[0].gc_calls_mean
+  t1 = pi[0].gc_time_mean
+  c2 = pi[1].gc_calls_mean
+  t2 = pi[1].gc_time_mean
+  factor = (c1-c2==0) ? 1 : c1/c2
+
+  if narrow
+    printf "\n%-4s %9s %9s %7s %7s %8s %8s %6s\n",
+           "GC:", "c1 real", "c2 real", "c1 #gc", "c2 #gc", "c1 gc%", "c2 gc%", "c1/c2"
+    printf "%-4s %9.5f %9.5f %7.1f %7.1f %8.2f %8.2f %6.2f\n",
+           "", t1, t2, c1, c2, (t1/total_time1)*100, (t2/total_time2)*100, factor
+  else
+    printf "\n%-32s %9s %9s %7s %7s %8s %8s %6s\n",
+           "garbage collection stats", "c1 real", "c2 real", "c1 #gc", "c2 #gc", "c1 gc%", "c2 gc%", "c1/c2"
+    printf "%-32s %9.5f %9.5f %7.1f %7.1f %8.2f %8.2f %6.2f\n",
+           "", t1, t2, c1, c2, (t1/total_time1)*100, (t2/total_time2)*100, factor
+  end
+end
+
+if narrow and !skip_urls
+  puts "\nurls:"
+  keys.each_with_index do |k, index|
+    printf "%2d: %s\n", (index+1) , k
+  end
+end
+
+__END__
+
+### Local Variables: ***
+### mode:ruby ***
+### End: ***
+
+#    Copyright (C) 2005, 2006  Stefan Kaes
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

data/script/perf_comp_gc

@@ -0,0 +1,109 @@
+#!/usr/bin/env ruby
+
+options = ""
+if ARGV.length!=2
+  puts "usage: perf_diff_gc file1 file2"
+  exit 1
+end
+files=[]
+ARGV.each do |arg|
+  fn = arg
+  fn = fn.sub(/^\/([cdefgh])(\/)/, '\1:\2') if RUBY_PLATFORM =~ /win32/
+  begin
+    if File.stat(fn).readable?
+      files << File.open(fn)
+    else
+      print "file #{fn} is unreadable\n"
+      exit 1
+    end
+  rescue
+    print "file #{fn} does not exist\n"
+    exit 1
+  end
+end
+
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+require 'railsbench/gc_info'
+
+printf "GC data file1: #{File.expand_path(files[0].path)}\n"
+printf "GC data file2: #{File.expand_path(files[1].path)}\n\n"
+
+gcis = files.map{|file| GCInfo.new(file)}
+files.each{|file| file.close}
+
+printf "number of requests: %d/%d\n", gcis[0].num_requests, gcis[1].num_requests
+
+printf "                         %12s %12s %7s\n", "c1", "c2", "c1/c2"
+printf "collections            : %12.2f %12.2f %7.2f\n",
+       (n1 = gcis[0].collections),
+       (n2 = gcis[1].collections),
+       n1.to_f/n2
+printf "garbage total          : %12.2f %12.2f %7.2f\n",
+       (n1 = gcis[0].garbage_produced),
+       (n2 = gcis[1].garbage_produced),
+       n1.to_f/n2
+printf "gc time total (sec)    : %12.2f %12.2f %7.2f\n",
+       (n1 = gcis[0].time_total.to_f/1000),
+       (n2 = gcis[1].time_total.to_f/1000),
+       n1/n2
+printf "garbage per request    : %12.2f %12.2f %7.2f\n",
+       (n1 = gcis[0].garbage_produced.to_f/gcis[0].num_requests),
+       (n2 = gcis[1].garbage_produced.to_f/gcis[1].num_requests),
+       n1/n2
+printf "requests per collection: %12.2f %12.2f %7.2f\n",
+       (n1 = gcis[0].num_requests.to_f/gcis[0].collections),
+       (n2 = gcis[1].num_requests.to_f/gcis[1].collections),
+       n1/n2
+
+printf "\n             %12s %12s %9s %9s %9s %9s\n",
+       "c1 mean", "c2 mean", "c1 min", "c2 min", "c1 max", "c2 max"
+
+number_format = "%12.2f %12.2f %9d %9d %9d %9d"
+  
+printf "gc time(ms): #{number_format}\n",
+       gcis[0].time_mean, gcis[1].time_mean,
+       gcis[0].time_min, gcis[1].time_min,
+       gcis[0].time_max, gcis[1].time_max
+   
+printf "heap slots : #{number_format}\n",
+       gcis[0].processed_mean, gcis[1].processed_mean,
+       gcis[0].processed_min, gcis[1].processed_min,
+       gcis[0].processed_max, gcis[1].processed_max
+
+printf "live       : #{number_format}\n",
+       gcis[0].live_mean, gcis[1].live_mean,
+       gcis[0].live_min, gcis[1].live_min,
+       gcis[0].live_max, gcis[1].live_max
+  
+printf "freed      : #{number_format}\n",
+       gcis[0].freed_mean, gcis[1].freed_mean,
+       gcis[0].freed_min, gcis[1].freed_min,
+       gcis[0].freed_max, gcis[1].freed_max
+  
+printf "freelist   : #{number_format}\n",
+       gcis[0].freelist_mean, gcis[1].freelist_mean,
+       gcis[0].freelist_min, gcis[1].freelist_min,
+       gcis[0].freelist_max, gcis[1].freelist_max
+
+
+__END__
+
+### Local Variables: ***
+### mode:ruby ***
+### End: ***
+
+#    Copyright (C) 2005, 2006  Stefan Kaes
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

data/script/perf_diff

@@ -0,0 +1,48 @@
+#!/bin/sh
+# set -x
+
+if [ $# -lt 4 ]; then
+  echo "usage: perf_diff iterations common-options options1 options4 [conf-name1] [conf-name2]"
+  echo "example: perf_diff 100 \"-bm=all\"  \"-log\" \"-nocache\" c1 c2"
+  exit 1
+fi
+
+BINDIR=`dirname $0`
+test -z "$RAILS_PERF_DATA" && RAILS_PERF_DATA=$HOME
+
+ITER="$1"
+COMMON="$2"
+OPT1="$3"
+OPT2="$4"
+DATE=`date +%m-%d`
+BENCHMARK=""
+for opt in $COMMON; do
+  case $opt in
+  -bm=*) BENCHMARK=${opt#-bm=};;
+  esac
+done
+
+if [ $# -gt 4 ]; then
+  FILE1="$RAILS_PERF_DATA/${DATE}.${BENCHMARK}.$5.txt"
+else
+  FILE1="$RAILS_PERF_DATA/perf_run1.${BENCHMARK}.txt"
+fi
+if [ $# -gt 5 ]; then
+  FILE2="$RAILS_PERF_DATA/${DATE}.${BENCHMARK}.$6.txt"
+else
+  FILE2="$RAILS_PERF_DATA/perf_run2.${BENCHMARK}.txt"
+fi
+
+unset RUBY_GC_STATS
+
+PERF_OPTIONS="$ITER $COMMON $OPT1"
+RAILS_BENCHMARK_FILE=$FILE1 $BINDIR/perf_loop $PERF_OPTIONS || exit 1
+ruby $BINDIR/perf_times $FILE1
+echo;echo
+
+PERF_OPTIONS="$ITER $COMMON $OPT2"
+RAILS_BENCHMARK_FILE=$FILE2 $BINDIR/perf_loop $PERF_OPTIONS || exit 1
+ruby $BINDIR/perf_times $FILE2
+echo;echo
+
+ruby $BINDIR/perf_comp -narrow -skip_urls $FILE1 $FILE2

data/script/perf_diff_gc

@@ -0,0 +1,89 @@
+#!/bin/sh
+# set -x
+
+if [ $# -lt 4 ]; then
+  echo "usage: perf_diff_gc iterations options1 options2 [conf-name1] [conf-name2]"
+  echo "example: perf_diff_gc 100 \"-bm=default\" \"-log\" \"-nocache\" c1 c2"
+  exit 1
+fi
+
+BINDIR=`dirname $0`
+test -z "$RAILS_PERF_DATA" && RAILS_PERF_DATA=$HOME
+
+ITER="$1"
+COMMON="$2"
+OPT1="$3"
+OPT2="$4"
+BENCHMARK=""
+WARMUP="-warmup"
+
+DATE=`date +%m-%d`
+
+
+if [ $# -gt 4 ]; then
+  FILE1="$RAILS_PERF_DATA/${DATE}${BENCHMARK}.$5.gc.txt"
+else
+  FILE1="$RAILS_PERF_DATA/perf_run1${BENCHMARK}.gc.txt"
+fi
+if [ $# -gt 5 ]; then
+  FILE2="$RAILS_PERF_DATA/${DATE}${BENCHMARK}.$6.gc.txt"
+else
+  FILE2="$RAILS_PERF_DATA/perf_run2${BENCHMARK}.gc.txt"
+fi
+
+OLD_RUBY_HEAP_MIN_SLOTS=$RUBY_HEAP_MIN_SLOTS
+OLD_RUBY_GC_MALLOC_LIMIT=$RUBY_GC_MALLOC_LIMIT
+OLD_RUBY_HEAP_FREE_MIN=$RUBY_HEAP_FREE_MIN
+
+PERF_OPTIONS="$ITER $WARMUP $COMMON $OPT1"
+
+patched_gc="no"
+for opt in $COMMON $OPT1; do
+  case $opt in
+  -gc=*) patched_gc=${opt#-gc=};;
+  esac
+done
+
+if [ "${patched_gc}" = "no" ]; then
+  unset RUBY_HEAP_MIN_SLOTS RUBY_GC_MALLOC_LIMIT RUBY_HEAP_FREE_MIN
+else
+  . $RAILS_ROOT/config/${patched_gc}.gc
+  export RUBY_HEAP_MIN_SLOTS RUBY_GC_MALLOC_LIMIT RUBY_HEAP_FREE_MIN
+fi
+
+echo "benchmarking GC performance with options $PERF_OPTIONS"
+echo
+
+(RUBY_GC_STATS=1 RUBY_GC_DATA_FILE=$FILE1 ruby $BINDIR/run_urls $PERF_OPTIONS >/dev/null) \
+ && ruby $BINDIR/perf_times_gc $FILE1
+echo;echo
+
+PERF_OPTIONS="$ITER $WARMUP $COMMON $OPT2"
+
+patched_gc="no"
+for opt in $COMMON $OPT2; do
+  case $opt in
+  -gc=*) patched_gc=${opt#-gc=};;
+  esac
+done
+
+if [ "${patched_gc}" = "no" ]; then
+  unset RUBY_HEAP_MIN_SLOTS RUBY_GC_MALLOC_LIMIT RUBY_HEAP_FREE_MIN
+else
+  . $RAILS_ROOT/config/${patched_gc}.gc
+  export RUBY_HEAP_MIN_SLOTS RUBY_GC_MALLOC_LIMIT RUBY_HEAP_FREE_MIN
+fi
+
+echo "benchmarking GC performance with options $PERF_OPTIONS"
+echo
+
+(RUBY_GC_STATS=1 RUBY_GC_DATA_FILE=$FILE2 ruby $BINDIR/run_urls $PERF_OPTIONS >/dev/null) \
+ && ruby $BINDIR/perf_times_gc $FILE2
+echo;echo
+
+unset RUBY_HEAP_MIN_SLOTS RUBY_GC_MALLOC_LIMIT RUBY_HEAP_FREE_MIN
+
+echo "benchmark comparison data"
+echo
+
+ruby $BINDIR/perf_comp_gc $FILE1 $FILE2