autobench 0.0.1alpha1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2c5f2e2376802ee01016f1887d3ff724f3547ee0
+  data.tar.gz: 2e96794b47c4a29d977cbd4bc47735a765090935
+SHA512:
+  metadata.gz: cf4ff282040334a17e19b8097f67d38809933223eabe5b5c528a382a4ba3fc56dec0262cf20f9dbb028c3c00ce7ccdceecf0e7feb302cd66b2949a780357f005
+  data.tar.gz: 80d4181decbd594c9d7b664fbf3bd7ccaa40ae352305fe18f09b72f372f6f513bb2f581c1f8552111f05a821fa24a33bf472de143fcd967fb10ea1a676de85b6

data/Gemfile

@@ -0,0 +1,16 @@
+# A sample Gemfile
+source "https://rubygems.org"
+
+gem "httperfrb", "~> 0.3.11"
+
+group :test do
+  gem "rake", "~> 10.1.0"
+  gem "minitest", "~> 5.0.5"
+end
+
+group :development do
+  gem "pry"
+  gem "pry-doc"
+end
+
+# vim:ft=ruby:

data/README.md

@@ -0,0 +1,123 @@
+# autobench
+
+> NOTE: This is still very much under development and I've written the README to define how I plan for it to work.
+>
+> TODOs:
+>
+> * Currently, yslow.js and phantomas do not work, unless
+>   you're running this from within the root directoy. I'm
+>   still trying to figure out why.
+
+### TODO: Installation
+
+    gem install autobench
+
+    # or...
+
+    bundle init
+    echo "gem 'autobench'" >> Gemfile
+    bundle install --path vendor/bundle
+
+### Usage
+
+#### `autobench`
+
+    Usage: autobench [options]
+
+    Required Arguments:
+        -c, --config FILENAME            configuration file
+
+    Optional Arguments:
+
+      Note: server, port, uri and runs overide what's in
+      your configuation file.
+
+        -s, --server SERVER              target server
+        -p, --port PORT                  target port
+        -u, --uri URI                    target uri
+        -i, --runs RUNS                  itterations against target
+        -f, --format [json|yaml|csv]     raw output format
+        -r, --results [dirname]          results directory (default: /tmp)
+
+    Module Exclusion:
+        -R, --without-render             Do not include render (httperf).
+        -Y, --without-yslow              Do not include yslow (yslow.js).
+        -C, --without-client             Do not include client (phantomas).
+
+    Misc. Arguments:
+        -q, --quiet                      Print no things.
+        -h, --help                       Show this message
+
+#### `autobench-config`
+
+The purpose of this is to generate a base configuration for your page, containing
+all possible values in their current state.
+
+> Note: I recommend looking over each configuration and removing those you don't
+> care about. Also, you should bump most up just a bit to account for standard
+> network variance.
+
+    Usage: autobench-config [options]
+
+    Required Arguments:
+        -o, --output FILENAME            configuration output file
+
+    Optional Arguments:
+
+      Note: server, port, uri and runs overide defaults:
+      - server: 'localhost'
+      - port:   80
+      - uri:    '/'
+      - runs:   9
+
+        -s, --server SERVER              target server
+        -p, --port PORT                  target port
+        -u, --uri URI                    target uri
+        -i, --runs RUNS                  itterations against target
+
+    Module Exclusion:
+        -R, --without-render             Do not include render (httperf).
+        -Y, --without-yslow              Do not include yslow (yslow.js).
+        -C, --without-client             Do not include client (phantomas).
+
+    Misc. Arguments:
+        -h, --help                       Show this message
+
+### Minitest Integration
+
+> #### TODO
+> Advanced Examples
+
+    require "minitest/autorun"
+    require "autobench"
+
+    class TestBenchmarks < Minitest::Test
+        def setup
+            @autobench = Autobench.new( "./path/to/config.yml", { ... config overides ... } )
+        end
+
+        def test_autobench_render
+            @autobench.render.benchmark
+            assert @autobench.render["connection_time_median"] < 500 # ms
+
+            # or...
+            assert @autobench.render.passed?
+        end
+
+        def test_autobench_yslow
+            @autobench.yslow.benchmark
+            assert @autobench.yslow["overall"] >= 95 # score
+
+            # or...
+            assert @autobench.yslow.passed?
+        end
+
+        def test_autobench_client
+            @autobench.client.benchmark
+            assert @autobench["requests"] < 100 # connections
+
+            # or...
+            assert @autobench.client.passed?
+        end
+    end
+

data/bin/autobench

@@ -0,0 +1,180 @@
+#!/usr/bin/env ruby
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')))
+
+################################################################################
+# Really, Autobench was designed to be hooked up to test frameworks,
+# however, here's a hacky/ugly script to run it via command line.
+################################################################################
+
+require 'optparse'
+require 'benchmark'
+require 'json'
+require 'yaml'
+require 'csv'
+require 'autobench'
+
+# opts
+@quiet     = false
+@format    = "json"
+@resdir    = "/tmp"
+options    = {}
+benchmarks = {
+  # defaults
+  :render => true,
+  :yslow  => true,
+  :client => true
+}
+optparser = OptionParser.new do |opts|
+  opts.banner = "Usage: autobench [options]"
+
+  opts.separator ""
+  opts.separator "Required Arguments:"
+
+  opts.on("-c", "--config FILENAME", "configuration file") do |p|
+    options["config"] = p
+  end
+
+  opts.separator ""
+  opts.separator "Optional Arguments:"
+  opts.separator ""
+  opts.separator "  Note: server, port, uri and runs overide what's in"
+  opts.separator "  your configuation file."
+  opts.separator ""
+
+
+  opts.on("-s", "--server SERVER", "target server") do |p|
+    options["server"] = p
+  end
+
+  opts.on("-p", "--port PORT", Integer, "target port") do |p|
+    options["port"] = p
+  end
+
+  opts.on("-u", "--uri URI", "target uri") do |p|
+    options["uri"] = p
+  end
+
+  opts.on("-i", "--runs RUNS", Integer, "itterations against target") do |p|
+    options["runs"] = p
+  end
+
+  opts.on("-f", "--format [json|yaml|csv]", "raw output format") do |p|
+    @format = p
+  end
+
+  opts.on("-r", "--results [dirname]", "results directory (default: /tmp)") do |p|
+    @resdir = File.expand_path(p)
+  end
+
+  opts.separator ""
+  opts.separator "Module Exclusion:"
+  opts.on("-R", "--without-render", "Do not include render (httperf).") do
+    benchmarks[:render] = false
+  end
+
+  opts.on("-Y", "--without-yslow", "Do not include yslow (yslow.js).") do
+    benchmarks[:yslow] = false
+  end
+
+  opts.on("-C", "--without-client", "Do not include client (phantomas).") do
+    benchmarks[:client] = false
+  end
+
+  opts.separator ""
+  opts.separator "Misc. Arguments:"
+  opts.on("-q", "--quiet", "Print no things.") do
+    @quiet = true
+  end
+
+  opts.on("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end
+optparser.parse!
+
+unless options.has_key?("config")
+  puts optparser
+  exit
+end
+
+def say msg
+  unless @quiet
+    puts msg
+  end
+end
+
+# utils
+
+def dump run
+    outfile = File.join(@resdir, "autobench-#{run}-#{Time.now.to_i}.#{@format.gsub('yaml','yml')}")
+    results = @autobench.send(run).full_results
+    File.open(outfile, "w") do |f|
+      case @format
+      when "json"
+        f.write(JSON.pretty_generate(results))
+      when "yaml"
+        f.write(results.to_yaml)
+      when "csv"
+        f.write(
+          CSV.generate { |csv|
+            csv << results.keys
+            csv << results.values
+          }
+        )
+      else
+        raise "unknown format `#{@format}`, try `yaml`, `json` or `csv`"
+        exit 556
+      end
+    end
+    say "=> Raw results (#{@format}): #{outfile}"
+end
+
+def handle_results test_text, run, time_taken
+  @total_time = @total_time + time_taken
+
+  say "Ran #{test_text}"
+  say "=> Duration: #{time_taken}"
+
+  if @autobench.send(run).thresholds?
+    say "=> Successes:"
+    say @autobench.send(run).successes
+
+    say "=> Failures:"
+    say @autobench.send(run).failures
+
+    if @autobench.send(run).passed?
+      say "=> PASS!"
+    else
+      @exit_status = @exit_status + 1
+      say "=> FAIL!"
+    end
+  end
+  dump(run)
+  say " "
+end
+
+# action
+@autobench   = Autobench.new(options.delete("config"), options)
+@exit_status = 0
+@total_time  = 0
+
+{ :render => "Render (httperf)",
+  :yslow  => "YSlow (yslow.js)",
+  :client => "Client (phantomas)"
+}.each do |run, desc|
+  if benchmarks[run]
+    render_time = Benchmark.realtime do
+      @autobench.send(run).benchmark
+    end
+    handle_results(desc, run, render_time)
+  end
+end
+
+say "Complete!"
+say "=> Duration: #{@total_time}"
+say "=> #{(@exit_status == 0 ? "Success" : "Failure")}!"
+
+exit @exit_status
+
+# vim: ft=ruby

data/bin/autobench-config

@@ -0,0 +1,162 @@
+#!/usr/bin/env ruby
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')))
+
+################################################################################
+# Really, Autobench was designed to be hooked up to test frameworks,
+# however, here's a hacky/ugly script to run it via command line.
+################################################################################
+
+require 'optparse'
+require 'yaml'
+require 'benchmark'
+require 'autobench'
+
+# opts
+options    = {
+  "server" => "localhost",
+  "port"   => 80,
+  "uri"    => "/",
+  "runs"   => 9
+}
+benchmarks = {
+  # defaults
+  :render => true,
+  :yslow  => true,
+  :client => true
+}
+optparser = OptionParser.new do |opts|
+  opts.banner = "Usage: autobench-config [options]"
+
+  opts.separator ""
+  opts.separator "Required Arguments:"
+
+  opts.on("-o", "--output FILENAME", "configuration output file") do |p|
+    @output = p
+  end
+
+  opts.separator ""
+  opts.separator "Optional Arguments:"
+  opts.separator ""
+  opts.separator "  Note: server, port, uri and runs overide defaults:"
+  opts.separator "  - server: 'localhost'"
+  opts.separator "  - port:   80"
+  opts.separator "  - uri:    '/'"
+  opts.separator "  - runs:   9"
+  opts.separator ""
+
+
+  opts.on("-s", "--server SERVER", "target server") do |p|
+    options["server"] = p
+  end
+
+  opts.on("-p", "--port PORT", Integer, "target port") do |p|
+    options["port"] = p
+  end
+
+  opts.on("-u", "--uri URI", "target uri") do |p|
+    options["uri"] = p
+  end
+
+  opts.on("-i", "--runs RUNS", Integer, "itterations against target") do |p|
+    options["runs"] = p
+  end
+
+  opts.separator ""
+  opts.separator "Module Exclusion:"
+  opts.on("-R", "--without-render", "Do not include render (httperf).") do
+    benchmarks[:render] = false
+  end
+
+  opts.on("-Y", "--without-yslow", "Do not include yslow (yslow.js).") do
+    benchmarks[:yslow] = false
+  end
+
+  opts.on("-C", "--without-client", "Do not include client (phantomas).") do
+    benchmarks[:client] = false
+  end
+
+  opts.separator ""
+  opts.separator "Misc. Arguments:"
+
+  opts.on("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end
+optparser.parse!
+
+unless defined?(@output)
+  puts optparser
+  exit
+end
+
+def say msg
+  puts msg
+end
+
+# utils
+
+def dump_data
+    File.open(@output, "w") do |f|
+      f.write(@data.to_yaml)
+    end
+    say "=> Results: #{@output}"
+end
+
+def handle_results test_text, run, time_taken
+  @total_time = @total_time + time_taken
+
+  say "Collected data: #{test_text}"
+  say "=> Duration: #{time_taken}"
+  say " "
+
+  if run == :yslow
+    @data["thresholds"]["yslow"] = @autobench.yslow.full_results.inject({}){ |memo,(k,v)| memo[ @autobench.yslow.mappings.select { |key, val| val == k }.keys.first ] = v; memo }
+  else
+    @data["thresholds"][run.to_s] = @autobench.send(run).full_results
+  end
+
+end
+
+# action
+@autobench   = Autobench.new(options)
+@exit_status = 0
+@total_time  = 0
+
+@data = options.merge({
+  "thresholds" => {
+    "render"   => {},
+    "yslow"    => {},
+    "client"   => {}
+  }
+})
+
+%w[ phantomjs node httperf ].each do |bin_name|
+  if (bin_path = %x{which #{bin_name}}.strip) && bin_path != ""
+    @data["paths"] ||= {}
+    @data["paths"][bin_name] = bin_path
+  else
+    say "WARNING: #{bin_name} not found."
+  end
+end
+
+{ :render => "Render (httperf)",
+  :yslow  => "YSlow (yslow.js)",
+  :client => "Client (phantomas)"
+}.each do |run, desc|
+  if benchmarks[run]
+    render_time = Benchmark.realtime do
+      @autobench.send(run).benchmark
+    end
+    handle_results(desc, run, render_time)
+  end
+end
+
+dump_data
+
+say "Complete!"
+say "=> Duration: #{@total_time}"
+
+exit
+
+# vim: ft=ruby