autobench 0.0.1alpha1

data/lib/autobench.rb

@@ -0,0 +1,28 @@
+require 'autobench/common'
+require 'autobench/config'
+require 'autobench/render'
+require 'autobench/yslow'
+require 'autobench/client'
+
+class Autobench
+  attr_accessor :config
+
+  LIB_DIR ||= File.dirname(__FILE__)
+  def initialize config, overides={}
+    @config = Autobench::Config.new(config, overides)
+  end
+
+  def render
+    @render ||= Autobench::Render.new(@config)
+  end
+
+  def yslow
+    @yslow  ||= Autobench::YSlow.new(@config)
+  end
+
+  def client
+    @client ||= Autobench::Client.new(@config)
+  end
+end
+
+# vim: ft=ruby:

data/lib/autobench/client.rb

@@ -0,0 +1,78 @@
+class Autobench
+  class Client
+    include Common
+    def initialize config
+      @config       = config
+      @thresholds   = @config["thresholds"]["client"] rescue []
+      @full_results = []
+      @failures     = []
+      @successes    = []
+    end
+
+    def benchmark
+      begin
+        @full_results = JSON.parse(%x{#{command}}.strip)
+        @thresholds.each do |key,threshold|
+          if fetch(key) > threshold
+            @failures.push("#{key} is #{fetch(key)}, threshold is #{threshold}")
+          else
+            @successes.push("#{key} is #{fetch(key)}, threshold is #{threshold}")
+          end
+        end
+        @full_results
+      rescue
+        puts "COMMAND FAILED: #{command}"
+        raise
+      end
+    end
+
+    def [](key)
+      return fetch(key)
+    end
+
+    def fetch(key)
+      return median(raw_results.map {|i| i[key] })
+    end
+
+    # backdoor hook to @full_results
+    def raw_results
+      raise "missing benchmarks" unless @full_results
+      @full_results
+    end
+
+    # generate result set that is only medians by default
+    def full_results
+      r = {}
+      raw_results.first.keys.each do |key|
+        r[key] = fetch(key)
+      end
+      return r
+    end
+
+    private
+    def command
+      "cd #{@config.phantomas} && #{@config.node} ./run-multiple.js #{modules} --url=#{url} --runs=#{@config["runs"]} --format=json"
+    end
+
+    def url
+      "http://#{@config['server']}#{(@config['port'] == 80 ? "" : ":#{@config['port']}")}#{@config['uri']}"
+    end
+
+    def modules
+      mod_str = ""
+      if @config.has_key?("phantomas") && @config["phantomas"].has_key?("modules")
+        mod_str << "--modules="+@config["phantomas"]["modules"].join(",")
+      end
+      return mod_str
+    end
+
+    def median(array)
+      sorted = array.sort
+      len = sorted.length
+      return (sorted[(len - 1) / 2] + sorted[len / 2]) / 2.0
+    end
+  end
+end
+
+# vim: ft=ruby:
+

data/lib/autobench/common.rb

@@ -0,0 +1,49 @@
+class Autobench
+  module Common
+    def [](key)
+      full_results[key.to_sym]
+    end
+
+    def clean_keys results, ignored
+      raise "missing benchmarks" unless @full_results
+      results.each_key do |key|
+        results.delete(key) if ignored.include?(key)
+      end
+      return results
+    end
+
+    def failures
+      return "none" if passed?
+      out = ""
+      @failures.each_with_index do |f,i|
+        out << "[#{i+1}] #{f}\n"
+      end
+      return out
+    end
+
+    def successes
+      raise "missing benchmarks" unless @full_results
+      return "none" if @successes.empty?
+      out = ""
+      @successes.each_with_index do |f,i|
+        out << "[#{i+1}] #{f}\n"
+      end
+      return out
+    end
+
+    def passed?
+      raise "missing benchmarks" unless @full_results
+      return @failures.empty?
+    end
+
+    def failed?
+      !passed?
+    end
+
+    def thresholds?
+      return !@thresholds.empty?
+    end
+
+  end
+end
+

data/lib/autobench/config.rb

@@ -0,0 +1,62 @@
+require "yaml"
+require "json"
+
+class Autobench
+  LIB_DIR ||= File.expand_path('..', File.dirname(__FILE__))
+  class Config
+    attr_accessor :basedir
+    def initialize config, overides={}
+      @config = if config.is_a?(String)
+                  YAML.load_file(config).merge(overides)
+                elsif config.is_a?(Hash)
+                  config.merge(overides)
+                else
+                  raise "invalid config"
+                end
+    end
+
+    def phantomas
+      default = File.join(::Autobench::LIB_DIR, "phantomas")
+      begin
+        return default if @config["paths"]["phantomas"].nil?
+        return @config["paths"]["phantomas"]
+      rescue NoMethodError
+        return default
+      end
+   end
+
+    def yslow
+      default = ::Autobench::LIB_DIR
+      begin
+        return default if @config["paths"]["yslow"].nil?
+        return @config["paths"]["yslow"]
+      rescue NoMethodError
+        return default
+      end
+    end
+
+    %w{ httperf node phantomjs }.each do |bin|
+      define_method(bin) do
+        return @config["paths"][bin] rescue bin
+      end
+    end
+    #def httperf
+      #return @config["paths"]["httperf"] rescue "httperf"
+    #end
+
+    #def node
+      #return @config["paths"]["node"] rescue "node"
+    #end
+
+    #def phantomjs
+      #return @config["paths"]["phantomjs"] rescue "phantomjs"
+    #end
+
+
+    def method_missing(meth, *args, &block)
+      @config.send(meth.to_sym, *args, &block) rescue super
+    end
+  end
+end
+
+# vim: ft=ruby:

data/lib/autobench/render.rb

@@ -0,0 +1,102 @@
+require 'httperf'
+class Autobench
+  LIB_DIR ||= File.expand_path('..', File.dirname(__FILE__))
+  class Render
+    include Common
+    def initialize config
+      @config     = config
+      @thresholds = config["thresholds"]["render"] rescue []
+      @failures   = []
+      @successes  = []
+      setup_httperf_configuration(config)
+    end
+
+    def benchmark
+      httperf       = HTTPerf.new(@httperf_config, @config.httperf)
+      httperf.parse = true
+
+      @full_results = httperf.run.inject({}){|memo,(k,v)| memo[k.to_s] = v; memo}
+                                  # keys: sym -> str
+
+      @thresholds.each do |key,threshold|
+        if higher_is_better.include?(key)
+          if (full_results[key].to_f < threshold.to_f)
+            @failures.push("#{key} is #{full_results[key].to_f}, threshold is #{threshold.to_f}")
+          else
+            @successes.push("#{key} is #{full_results[key].to_f}, threshold is #{threshold.to_f}")
+          end
+        else
+          if (full_results[key].to_f > threshold.to_f)
+            @failures.push("#{key} is #{full_results[key].to_f}, threshold is #{threshold.to_f}")
+          else
+            @successes.push("#{key} is #{full_results[key].to_f}, threshold is #{threshold.to_f}")
+          end
+        end
+      end
+
+      ignored_keys.each do |key|
+        @full_results.delete(key)
+      end
+
+      @full_results
+    end
+
+    def full_results
+      return clean_keys(@full_results, %w[ command connection_times ])
+    end
+
+    private
+    def setup_httperf_configuration config
+      @httperf_config = {} # defaults
+
+      if config.has_key?("httperf")
+        puts config["httperf"]
+        @httperf_config.merge!(config["httperf"])
+      end
+
+      # all required
+      %w{ server port uri }.each do |key|
+        raise "Autobench::Render missing `#{key}`." unless config.has_key?(key)
+        @httperf_config[key] = config[key]
+      end
+
+      if config.has_key?("runs")
+        @httperf_config["num-conns"] = config["runs"]
+      end
+
+      @httperf_config.merge!(httperf_forced_options)
+    end
+
+    def httperf_forced_options
+      { "verbose"   => true }
+    end
+
+    def higher_is_better
+      %w[
+        connection_rate_per_sec
+        request_rate_per_sec
+        reply_status_2xx
+        total_connections
+        total_requests
+        total_replies
+      ]
+    end
+    def ignored_keys
+      %w[
+        max_connect_burst_length
+        cpu_time_system_pct
+        cpu_time_total_pct
+        cpu_time_user_sec
+        cpu_time_system_sec
+        cpu_time_user_pct
+        reply_rate_min
+        reply_rate_avg
+        reply_rate_max
+        reply_rate_stddev
+        reply_rate_samples
+        connection_length
+      ]
+    end
+  end
+end
+# vim: ft=ruby:

data/lib/autobench/version.rb

@@ -0,0 +1,3 @@
+class Autobench
+  VERSION = '0.0.1alpha1'
+end

data/lib/autobench/yslow.rb

@@ -0,0 +1,75 @@
+require 'json'
+class Autobench
+  LIB_DIR ||= File.expand_path('..', File.dirname(__FILE__))
+  class YSlow
+    include Common
+    def initialize config
+      @config     = config
+      @thresholds = config["thresholds"]["yslow"] rescue []
+      @failures   = []
+      @successes  = []
+    end
+
+    def benchmark
+      begin
+        @full_results = JSON.parse(%x{#{command}}.strip)
+      rescue
+        raise "Error running: #{command}"
+      end
+      @thresholds.each do |key,threshold|
+        # U-G-L-Y
+        if key == "overall"
+          if full_results[mappings[key]].to_i < threshold.to_i
+            @failures.push("#{key} is #{full_results[mappings[key]]}, threshold is #{threshold}");
+          else
+            @successes.push("#{key} is #{full_results[mappings[key]]}, threshold is #{threshold}");
+          end
+        else
+          if full_results[mappings[key]].to_i > threshold.to_i
+            @failures.push("#{key} is #{full_results[mappings[key]]}, threshold is #{threshold}");
+          else
+            @successes.push("#{key} is #{full_results[mappings[key]]}, threshold is #{threshold}");
+          end
+        end
+      end
+      @full_results
+    end
+
+    def [](key)
+      return full_results[mappings[key]] if mappings.keys.include?(key)
+      return full_results[key]           if mappings.values.include?(key)
+      return nil
+    end
+
+    def full_results
+      return clean_keys(@full_results, ['u','i'])
+    end
+
+    def mappings
+      {
+        "overall"  => "o",
+        "loadtime" => "lt",
+        "requests" => "r",
+        "size"     => "w"
+      }
+    end
+
+    private
+    def command
+      "cd #{@config.yslow} && #{@config.phantomjs} ./yslow.js --info basic #{options} http://#{@config['server']}#{(@config.has_key?("port") ? ":#{@config['port']}" : "")}#{@config['uri']}"
+    end
+
+    def options
+      return "" unless @config.has_key?("yslow")
+
+      ret = []
+      %w{ ruleset ua headers viewport }.each do |key|
+        ret.push "--#{key} #{@config["yslow"][key]}" if @config["yslow"].has_key?(key)
+      end
+
+      return ret.join(" ")
+    end
+  end
+end
+
+# vim: ft=ruby:

data/lib/phantomas/README.md

@@ -0,0 +1,296 @@
+phantomas
+=========
+
+![GitHub Logo](http://upload.wikimedia.org/wikipedia/en/a/a5/Fantomas.jpg)
+
+PhantomJS-based modular web performance metrics collector.
+
+And why phantomas? Well, [because](http://en.wikipedia.org/wiki/Fantômas) :)
+
+## Requirements
+
+* [PhantomJS 1.7+](http://phantomjs.org/)
+* [NodeJS](http://nodejs.org) (for `run-multiple.js` script)
+
+## Installation
+
+```
+npm install phantomas
+```
+
+## Dependencies
+
+phantomas uses the following 3rd party libraries (located in `/lib` directory):
+
+* CommonJS modules from [phantomjs-nodify](https://github.com/jgonera/phantomjs-nodify) and nodejs source
+
+## Let's make Web a bit faster!
+
+* [Best Practices for Speeding Up Your Web Site](http://developer.yahoo.com/performance/rules.html) (by Yahoo!)
+* [Web Performance Best Practices](https://developers.google.com/speed/docs/best-practices/rules_intro) (by Google)
+
+## Contributors
+
+* [macbre](https://github.com/macbre)
+* [jmosney](https://github.com/jmosney)
+* All the [contributors](https://github.com/macbre/phantomas/graphs/contributors)
+
+## Usage
+
+### Single run
+
+``` bash
+./phantomas.js --url=https://github.com/macbre/phantomas  --verbose
+```
+
+#### Parameters
+
+* `--url` URL of the page to generate metrics for (required)
+* `--format=[json|csv|plain]` output format (``plain`` is the default one)
+* `--timeout=[seconds]` timeout for phantomas run (defaults to 15 seconds)
+* `--viewport=[width]x[height]` phantomJS viewport dimensions (1280x1024 is the default)
+* `--verbose` writes debug messages to the console
+* `--silent` don't write anything to the console
+* `--modules=[moduleOne],[moduleTwo]` run only selected modules
+* `--user-agent='Custom user agent'` provide a custom user agent (will default to something similar to ``phantomas/0.4 (PhantomJS/1.7.0; 64bit)``)
+
+### Multiple runs
+
+This helper script requires NodeJS.
+
+``` bash
+./run-multiple.js --url=https://github.com/macbre/phantomas  --runs=5
+```
+
+#### Parameters
+
+* `--url` URL of the page to generate metrics for (required)
+* `--runs` number of runs to perform (defaults to 3)
+* `--modules=[moduleOne],[moduleTwo]` run only selected modules
+
+## Features
+
+* Modular approach - each metric is generated by a separate "module"
+* phantomas "core" acts as an [events emitter](https://github.com/macbre/phantomas/wiki/Events) that each module can hook into
+* JSON and CSV as available output formats for easy integration with automated reporting / monitoring tools
+
+## Metrics
+
+_Current number of metrics: 64_
+
+Units:
+
+* ms for time
+* bytes for size
+
+``` 
+./phantomas.js --url=https://github.com/macbre/phantomas
+
+phantomas metrics for <https://github.com/macbre/phantomas>:
+
+* requests: 22
+* gzipRequests: 9
+* postRequests: 0
+* redirects: 0
+* notFound: 0
+* timeToFirstByte: 503
+* timeToLastByte: 520
+* bodySize: 675125
+* contentLength: 708206
+* ajaxRequests: 0
+* htmlCount: 1
+* htmlSize: 50728
+* cssCount: 2
+* cssSize: 174949
+* jsCount: 4
+* jsSize: 322932
+* imageCount: 12
+* imageSize: 49287
+* base64Count: 0
+* base64Size: 0
+* otherCount: 3
+* otherSize: 77229
+* cacheHits: 0
+* cacheMisses: 0
+* cachingNotSpecified: 3
+* cachingTooShort: 2
+* cachingDisabled: 0
+* headersCount: 265
+* headersSentCount: 65
+* headersRecvCount: 200
+* headersSize: 10205
+* headersSentSize: 3850
+* headersRecvSize: 6355
+* assetsNotGzipped: 0
+* assetsWithQueryString: 9
+* smallImages: 2
+* httpTrafficCompleted: 2460
+* domains: 5
+* DOMqueries: 39
+* DOMinserts: 19
+* jQuerySelectors: 0
+* jQueryOnDOMReadyFunctions: 0
+* cookiesSent: 0
+* cookiesRecv: 268
+* domainsWithCookies: 1
+* documentCookiesLength: 387
+* documentCookiesCount: 12
+* bodyHTMLSize: 47769
+* commentsSize: 452
+* hiddenContentSize: 15635
+* whiteSpacesSize: 2378
+* DOMelementsCount: 554
+* DOMelementMaxDepth: 13
+* iframesCount: 0
+* nodesWithInlineCSS: 5
+* imagesWithoutDimensions: 3
+* globalVariables: 18
+* localStorageEntries: 0
+* smallestResponse: 35
+* biggestResponse: 239453
+* fastestResponse: 43
+* slowestResponse: 984
+* medianResponse: 130.5
+* onDOMReadyTime: 482
+* windowOnLoadTime: 1716
+```
+
+### Requests monitor (core module)
+
+* requests: total number of HTTP requests made
+* gzipRequests: number of gzipped HTTP responses
+* postRequests: number of POST requests
+* redirects: number of HTTP redirects (either 301 or 302)
+* notFound: number of HTTP 404 responses
+* timeToFirstByte: time it took to receive the first byte of the first response
+* timeToLastByte: time it took to receive the last byte of the first response
+* bodySize: size of the content of all responses
+* contentLength: size of the content of all responses (based on ``Content-Length`` header)
+* httpTrafficCompleted: time it took to receive the last byte of the last HTTP response
+
+### AJAX requests
+
+* ajaxRequests: number of AJAX requests
+
+### Assets types
+
+* htmlCount: number of html responses
+* htmlSize: size of html responses
+* cssCount: number of css responses
+* cssSize: size of css responses
+* jsCount: number of js responses
+* jsSize: size of js responses
+* imageCount: number of image responses
+* imageSize: size of image responses
+* base64Count: number of base64 encoded "responses" (no HTTP request was made)
+* base64Size: size of base64 encoded "responses"
+* otherCount: number of other responses
+* otherSize: size of other responses
+
+### Cache Hits
+
+_Metrics are calculated based on ``X-Cache`` header added by Varnish  / Squid servers._
+
+* cacheHits: number of cache hits
+* cacheMisses: number of cache misses
+
+### Headers
+
+* headersCount: number of requests and responses headers
+* headersSentCount: number of headers sent in requests
+* headersRecvCount: number of headers received in responses
+* headersSize: size of all headers
+* headersSentSize: size of sent headers
+* headersRecvSize: size of received headers
+
+### Domains
+
+* domains: number of domains used to fetch the page
+
+### Cookies
+
+* cookiesSent: length of cookies sent in HTTP requests
+* cookiesRecv: length of cookies received in HTTP responses
+* domainsWithCookies: number of domains with cookies set
+* documentCookiesLength: length of `document.cookie`
+* documentCookiesCount: number of cookies in `document.cookie`
+
+### DOM complexity
+
+* globalVariables: number of JS globals variables
+* bodyHTMLSize: the size of body tag content
+* commentsSize: the size of HTML comments on the page
+* hiddenContentSize: the size of content of hidden elements on the page (with CSS ``display: none``)
+* whiteSpacesSize: the size of text nodes with whitespaces only
+* DOMelementsCount: total number of HTML element nodes
+* DOMelementMaxDepth: maximum level on nesting of HTML element node
+* iframesCount: number of iframe nodes
+* nodesWithInlineCSS: number of nodes with inline CSS styling (with `style` attribute)
+* imagesWithoutDimensions: number of ``<img>`` nodes without both ``width`` and ``height`` attribute
+
+### DOM queries
+
+* DOMqueries: number of `document.getElementById` and `document.getElementsByClassName` calls
+* DOMinserts: number of DOM nodes inserts
+* jQuerySelectors: number of jQuery selectors calls (e.g. `$('#foo > .bar')`)
+* jQueryOnDOMReadyFunctions: number of functions bound to onDOMready event
+
+### Window performance
+
+* onDOMReadyTime: time it took to fire onDOMready event
+* windowOnLoadTime: time it took to fire window.load event
+
+### Requests statistics
+
+* smallestResponse: the size of the smallest response
+* biggestResponse: the size of the biggest response
+* fastestResponse: the time to the last byte of the fastest response
+* slowestResponse: the time to the last byte of the slowest response
+* medianResponse: median value of time to the last byte for all responses
+
+### localStorage
+
+* localStorageEntries: number of entries in local storage
+
+### Static assets
+
+* assetsNotGzipped: static assets that were not gzipped
+* assetsWithQueryString: static assets requested with query string (e.g. ?foo) in URL
+* smallImages: images smaller than 2 kB that can be base64 encoded
+
+### Caching
+
+* cachingNotSpecified: responses with no caching header sent (either `Cache-Control` or `Expires`)
+* cachingTooShort: responses with too short (less than a week) caching time
+* cachingDisabled: responses with caching disabled (`max-age=0`)
+
+## Notices
+
+phantomas apart from "raw" metrics data, when in `--verbose` mode, emits notices with more in-depth data:
+
+```
+> Caching period is less than a week for <https://ssl.google-analytics.com/ga.js> (set to 43200 s)
+> No caching specified for <https://secure.gaug.es/track.js>
+> Caching period is less than a week for <https://secure.gravatar.com/avatar/57548e3255bfa0e74afff98289dae839?s=140&d=https://a248.e.akamai.net/assets.github.com%2Fimages%2Fgravatars%2Fgravatar-user-420.png> (set to 300 s)
+> https://secure.gravatar.com/avatar/57548e3255bfa0e74afff98289dae839?s=140&d=https://a248.e.akamai.net/assets.github.com%2Fimages%2Fgravatars%2Fgravatar-user-420.png (IMAGE) served with query string
+> Requests per domain:
+>  github.com: 1 request(s)
+>  a248.e.akamai.net: 16 request(s)
+>  ssl.google-analytics.com: 2 request(s)
+>  secure.gaug.es: 2 request(s)
+>  secure.gravatar.com: 1 request(s)
+>
+> JavaScript globals (18): html5, Modernizr, moment, $, jQuery, $stats, jQuery18302483161953277886, GitHub, DateInput, clippyCopiedCallback, debug, _gaq, _gauges, CommandBar, stringDistance, fuzzyScore, _gat, gaGlobal
+>
+> The smallest response (0.03 kB): https://ssl.google-analytics.com/__utm.gif?utmwv=5.3.8&utms=1&utmn=396347331&utmhn=github.com&utmcs=UTF-8&utmsr=1024x768&utmvp=1024x1280&utmsc=32-bit&utmul=pl-pl&utmje=0&utmfl=-&utmdt=macbre%2Fphantomas%20%C2%B7%20GitHub&utmhid=1963809109&utmr=-&utmp=%2Fmacbre%2Fphantomas&utmac=UA-3769691-2&utmcc=__utma%3D1.1523233271.1353260190.1353260190.1353260190.1%3B%2B__utmz%3D1.1353260190.1.1.utmcsr%3D(direct)%7Cutmccn%3D(direct)%7Cutmcmd%3D(none)%3B&utmu=qB~
+> The biggest response (233.84 kB): https://a248.e.akamai.net/assets.github.com/assets/github-81433815e4751f68e04d42ec948cba14ab028c2d.js
+>
+> The fastest response (43 ms): https://a248.e.akamai.net/assets.github.com/images/modules/header/logov7@4x.png?1340659561
+> The slowest response (984 ms): https://secure.gravatar.com/avatar/57548e3255bfa0e74afff98289dae839?s=140&d=https://a248.e.akamai.net/assets.github.com%2Fimages%2Fgravatars%2Fgravatar-user-420.png
+```
+
+## For developers
+
+* [Project's wiki](https://github.com/macbre/phantomas/wiki)
+* Description of [events fired by phantomas core](https://github.com/macbre/phantomas/wiki/Events)
+* [TODO list](https://github.com/macbre/phantomas/wiki/TODO)