baracus 0.1

data/README

@@ -0,0 +1,38 @@
+httperf wrapper for benchmarking CouchDB
+
+Features:
+
+* Results sent to CouchDB and printed in TSV
+* Creates wsesslog files dynamically using random data for document writes and _all_docs for reads
+* Lots of knobs: session counts, document size, request rate, etc
+* httperf session work logs are attached to results for replaying later
+
+
+License:
+
+This code is available as Open Source Software under the MIT license.
+
+
+Dependencies:
+
+gem sources -a http://gems.opscode.com/gems
+gem install mixlib-config open4 json rest-client --no-ri --no-rdoc
+
+httperf installed (see suggestions below before installing)
+
+
+Usage:
+
+$ ./baracus.rb baracus.yml
+
+
+Suggestions:
+
+* Set "ulimit -n 65535" before running baracus
+* Change FD_SETSIZE to 65535 in /usr/include/bits/typesizes.h before compiling httperf
+* For tests with more than 1000 sessions increase MAX_SESSION_TEMPLATES in gen/wsesslog.c before compiling httperf
+
+Acknowlegdements:
+
+Details on httperf http://www.hpl.hp.com/research/linux/httperf/httperf-man-0.9.txt
+Some of the httperf output parsing code was taken from http://github.com/sv/httperf4r

data/bin/baracus

@@ -0,0 +1,100 @@
+#! /usr/bin/ruby
+
+## Copyright 2009, Joe Williams <joe@joetify.com>
+##
+## Permission is hereby granted, free of charge, to any person
+## obtaining a copy of this software and associated documentation
+## files (the "Software"), to deal in the Software without
+## restriction, including without limitation the rights to use,
+## copy, modify, merge, publish, distribute, sublicense, and/or sell
+## copies of the Software, and to permit persons to whom the
+## Software is furnished to do so, subject to the following
+## conditions:
+##
+## The above copyright notice and this permission notice shall be
+## included in all copies or substantial portions of the Software.
+##
+## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+## EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+## OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+## NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+## HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+## WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+## FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+## OTHER DEALINGS IN THE SOFTWARE.
+
+require 'rubygems'
+require 'rest_client'
+require 'json'
+require 'open4'
+include Open4
+require 'mixlib/config'
+require 'base64'
+require 'yaml'
+
+require 'lib/config'
+require 'lib/httperf'
+require 'lib/results'
+
+class Baracus
+
+  def self.get_url
+    if Baracus::Config.user && Baracus::Config.password
+      url = "http://#{Baracus::Config.user}:#{Baracus::Config.password}@#{Baracus::Config.host}:#{Baracus::Config.port}/#{Baracus::Config.db}"
+    else
+      url = "http://#{Baracus::Config.host}:#{Baracus::Config.port}/#{Baracus::Config.db}"
+    end
+    url
+  end
+
+
+  def self.main
+    date = Time.now
+
+    puts "Running #{Baracus::Config.bench_name}:"
+    puts "Database: http://#{Baracus::Config.host}:#{Baracus::Config.port}/#{Baracus::Config.db}"
+    puts "Rate: #{Baracus::Config.rate}"
+    puts "Sessions: #{Baracus::Config.sessions}"
+    puts "Writes: #{Baracus::Config.writes}"
+    puts "Reads: #{Baracus::Config.reads}"
+    puts "Document Size: #{Baracus::Config.doc_size}"
+    puts "Batch=ok: #{Baracus::Config.batchok}\n"
+
+
+    # create db
+    RestClient.put("#{get_url}", "")
+
+
+    # write test
+    write_wsesslog = Baracus::Httperf.create_write_wsesslog(date)
+    results = Baracus::Httperf.run_httperf(write_wsesslog)
+
+    puts "### #{Baracus::Config.bench_name} write results ###"
+    output = Baracus::Results.parse_results(results)
+    puts ""
+    puts Baracus::Results.format_results(output)
+    puts "#####################\n"
+
+    Baracus::Results.report_results(output, "writes", write_wsesslog, date)
+
+    puts "waiting for 60 seconds ...\n"
+    sleep 60
+
+    # read test
+    read_wsesslog = Baracus::Httperf.create_read_wsesslog(date)
+    results = Baracus::Httperf.run_httperf(read_wsesslog)
+
+    puts "### #{Baracus::Config.bench_name} read results ###"
+    output = Baracus::Results.parse_results(results)
+    puts ""
+    puts Baracus::Results.format_results(output)
+    puts "#####################"
+
+    Baracus::Results.report_results(output, "reads", read_wsesslog, date)
+
+    # clean up
+    RestClient.delete("#{get_url}")
+  end
+
+  main
+end

data/config/baracus.yml

@@ -0,0 +1,20 @@
+httperf: "/usr/bin/httperf"
+host: "localhost"
+port: "5984"
+user:
+password:
+db: "testdb"
+report_url: "http://localhost:5984/bench"
+name: "test123"
+config:
+  timeout: 30
+  sessions: 100
+  rate: 100
+  doc_size: 100
+  writes: 10
+  reads: 10
+  batchok: false
+info:
+  storage: "raid10, 4 disks, ebs"
+  anything: "stuff"
+  more: "foo"

data/lib/config.rb

@@ -0,0 +1,48 @@
+## Copyright 2009, Joe Williams <joe@joetify.com>
+##
+## Permission is hereby granted, free of charge, to any person
+## obtaining a copy of this software and associated documentation
+## files (the "Software"), to deal in the Software without
+## restriction, including without limitation the rights to use,
+## copy, modify, merge, publish, distribute, sublicense, and/or sell
+## copies of the Software, and to permit persons to whom the
+## Software is furnished to do so, subject to the following
+## conditions:
+##
+## The above copyright notice and this permission notice shall be
+## included in all copies or substantial portions of the Software.
+##
+## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+## EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+## OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+## NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+## HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+## WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+## FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+## OTHER DEALINGS IN THE SOFTWARE.
+
+class Baracus
+  class Config
+    baracus_config = YAML.load(File.open(ARGV[0]))
+    extend(Mixlib::Config)
+    configure do |c|
+      c[:httperf] = baracus_config["httperf"]
+      c[:host] = baracus_config["host"]
+      c[:port] = baracus_config["port"]
+      c[:db] = baracus_config["db"]
+      c[:report_url] = baracus_config["report_url"]
+      c[:bench_name] = baracus_config["name"]
+      c[:config] = baracus_config["config"]
+      c[:timeout] = baracus_config["config"]["timeout"]
+      c[:sessions] = baracus_config["config"]["sessions"]
+      c[:rate] = baracus_config["config"]["rate"]
+      c[:doc_size] = baracus_config["config"]["doc_size"]
+      c[:writes] = baracus_config["config"]["writes"]
+      c[:reads] = baracus_config["config"]["reads"]
+      c[:batchok] = baracus_config["config"]["batchok"]
+      c[:info] = baracus_config["info"]
+      c[:user] = baracus_config["user"]
+      c[:password] = baracus_config["password"]
+    end
+  end
+end

data/lib/httperf.rb

@@ -0,0 +1,116 @@
+## Copyright 2009, Joe Williams <joe@joetify.com>
+##
+## Permission is hereby granted, free of charge, to any person
+## obtaining a copy of this software and associated documentation
+## files (the "Software"), to deal in the Software without
+## restriction, including without limitation the rights to use,
+## copy, modify, merge, publish, distribute, sublicense, and/or sell
+## copies of the Software, and to permit persons to whom the
+## Software is furnished to do so, subject to the following
+## conditions:
+##
+## The above copyright notice and this permission notice shall be
+## included in all copies or substantial portions of the Software.
+##
+## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+## EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+## OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+## NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+## HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+## WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+## FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+## OTHER DEALINGS IN THE SOFTWARE.
+
+class Baracus
+  class Httperf
+
+    def self.create_write_wsesslog(date)
+      # create some random data for the doc writes
+      chars = ('a'..'z').to_a + ('A'..'Z').to_a
+      doc = (0...Baracus::Config.doc_size).collect { chars[rand(chars.length)] }.join
+
+      # use batch=ok for writes?
+      if Baracus::Config.batchok
+        batchok = "?batch=ok"
+      else
+        batchok = ""
+      end
+
+      # create the httperf wsesslog file
+      filename = "httperf_write_wsesslog_#{date.to_i}"
+      write_wsesslog = File.new(filename, "w")
+      (1..Baracus::Config.sessions).each do |session|
+        write_wsesslog.puts "# session #{session}"
+        (1..Baracus::Config.writes).each do |write|
+          write_wsesslog.puts "/#{Baracus::Config.db}/#{batchok} method=POST contents=\'{\"field\": \"#{doc}\"}\'"
+        end
+        write_wsesslog.puts ""
+      end
+      write_wsesslog.close
+      filename
+    end
+
+    def self.create_read_wsesslog(date)
+      # parse _all_docs to get all the doc ids for reads
+      if Baracus::Config.user && Baracus::Config.password
+        all_docs_json = RestClient.get("http://#{Baracus::Config.user}:#{Baracus::Config.password}@#{Baracus::Config.host}:#{Baracus::Config.port}/#{Baracus::Config.db}/_all_docs")
+      else
+        all_docs_json = RestClient.get("http://#{Baracus::Config.host}:#{Baracus::Config.port}/#{Baracus::Config.db}/_all_docs")
+      end
+
+      all_docs = JSON.parse(all_docs_json)
+      all_docs_rows = all_docs["rows"]
+
+      # create the httperf wsesslog file
+      filename = "httperf_read_wsesslog_#{date.to_i}"
+      read_wsesslog = File.new(filename, "w")
+      (1..Baracus::Config.sessions).each do |session|
+        read_wsesslog.puts "# session #{session}"
+        (1..Baracus::Config.reads).each do |read|
+          # pick a random doc
+          doc_index = rand(all_docs_rows.length)
+          read_wsesslog.puts "/#{Baracus::Config.db}/#{all_docs_rows[doc_index]["id"]} method=GET"
+        end
+        read_wsesslog.puts ""
+      end
+      read_wsesslog.close
+      filename
+    end
+
+
+    def self.run_httperf(wsesslog)
+      # for basic auth
+      if Baracus::Config.user && Baracus::Config.password
+        auth = Base64.encode64("#{Baracus::Config.user}:#{Baracus::Config.password}")
+        auth_switch = " --add-header 'Authorization: Basic #{auth}'"
+      else
+        auth_switch = ""
+      end
+
+      # run httperf with the wsesslog and write results to file
+      httperf_cmd = <<-EOH
+        #{Baracus::Config.httperf} \
+          --hog \
+          --rate=#{Baracus::Config.rate} \
+          --timeout=#{Baracus::Config.timeout} \
+          --send-buffer=4096 \
+          --recv-buffer=16384 \
+          --max-connections=4 \
+          --max-piped-calls=32 \
+          --server=#{Baracus::Config.host} \
+          --port=#{Baracus::Config.port} \
+          --wsesslog=#{Baracus::Config.sessions},0,#{wsesslog} #{auth_switch}
+      EOH
+
+      results = ""
+      status = popen4(httperf_cmd) do |pid, stdin, stdout, stderr|
+        stdout.each do |line|
+          results << line
+        end
+      end
+
+      results
+    end
+
+  end
+end

data/lib/results.rb

@@ -0,0 +1,101 @@
+## Copyright 2009, Joe Williams <joe@joetify.com>
+##
+## Permission is hereby granted, free of charge, to any person
+## obtaining a copy of this software and associated documentation
+## files (the "Software"), to deal in the Software without
+## restriction, including without limitation the rights to use,
+## copy, modify, merge, publish, distribute, sublicense, and/or sell
+## copies of the Software, and to permit persons to whom the
+## Software is furnished to do so, subject to the following
+## conditions:
+##
+## The above copyright notice and this permission notice shall be
+## included in all copies or substantial portions of the Software.
+##
+## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+## EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+## OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+## NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+## HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+## WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+## FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+## OTHER DEALINGS IN THE SOFTWARE.
+
+class Baracus
+  class Results
+
+    def self.parse_results(output)
+      results={}
+      output.each_line{ |line|
+         puts line
+         results['test_duration']=$1 if line=~/^Total: .*test-duration (\d+)/
+         results['replies']=$1.to_i if line=~/^Total: .*replies (\d+)/
+         results['con_rate']=$1 if line=~/^Connection rate: (\d+\.\d)/
+         results['req_rate']=$1 if line=~/^Request rate: (\d+\.\d)/
+         if line=~/^Reply rate .* min (\d+\.\d) avg (\d+\.\d) max (\d+\.\d) stddev (\d+\.\d)/
+           results['rep_rate_min']=$1
+           results['rep_rate_avg']=$2
+           results['rep_rate_max']=$3
+           results['rep_rate_stddev']=$4
+         end
+         results['rep_time']=$1 if line=~/^Reply time .* response (\d+\.\d)/
+         results['net_io']=$1 if line=~/^Net I\/O: (\d+\.\d)/
+         results['errors']=$1 if line=~/^Errors: total (\d+)/
+      }
+      if results['replies']==0
+        puts "Zero replies recieved,test invalid: rate #{results['rate']}\n"
+      else
+        results['percent_errors']=(100*results['errors'].to_f/results['replies'].to_f).to_s
+      end
+      results
+    end
+
+    def self.format_results(results)
+      tsv_results = results
+      line=""
+      header=['req_rate','con_rate','rep_rate_min','rep_rate_avg','rep_rate_max',
+              'rep_rate_stddev', 'rep_time','net_io','percent_errors']
+      header.each{|h| line << "#{h}\t"}
+      line << "\n"
+      header.each{|h| line << tsv_results[h] << "\t"}
+      line << "\n"
+      line
+    end
+
+    def self.report_results(results, type, wsesslog, date)
+      # convert strings to floats
+      results.each do |k, v|
+        results[k] = v.to_f
+      end
+
+      config = {}
+      config.update(Baracus::Config.config)
+      config.store(
+        "database", "http://#{Baracus::Config.host}:#{Baracus::Config.port}/#{Baracus::Config.db}"
+      )
+      results['config'] = config
+
+      info = {}
+      info.update(Baracus::Config.info)
+      version_json = RestClient.get("http://#{Baracus::Config.host}:#{Baracus::Config.port}/")
+      version = JSON.parse(version_json)
+      info.store("version", version["version"])
+      results['info'] = info
+      results['date'] = date
+      results_json = results.to_json
+
+      # send the results to couch and create the db if it doesnt exist
+      begin
+        reply_json = RestClient.put("#{Baracus::Config.report_url}/#{Baracus::Config.bench_name}_#{type}_#{date.to_i}", results_json, :content_type => "application/json")
+        reply = JSON.parse(reply_json)
+        wsesslog_file = File.read(wsesslog)
+        RestClient.put("#{Baracus::Config.report_url}/#{reply["id"]}/attachment?rev=#{reply["rev"]}", wsesslog_file, :content_type => "application/text")
+      rescue
+        RestClient.put("#{Baracus::Config.report_url}/", "")
+        retry
+      end
+
+    end
+
+  end
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification 
+name: baracus
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+platform: ruby
+authors: 
+- joe williams
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-02 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mixlib-config
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: open4
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: 
+email: joe@joetify.com
+executables: 
+- baracus
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- bin/baracus
+- lib/config.rb
+- lib/httperf.rb
+- lib/results.rb
+- config/baracus.yml
+- README
+has_rdoc: true
+homepage: http://github.com/joewilliams/baracus
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: httperf wrapper for couchdb
+test_files: []
+