pagespeed_grabber 0.0.1

data/Gemfile

@@ -0,0 +1,10 @@
+source :rubygems
+
+gem 'mechanize'
+gem 'fastercsv'
+
+group :dev do
+  gem 'rake'
+  gem 'rspec', '~>2'
+  gem 'jeweler'
+end

data/Gemfile.lock

@@ -0,0 +1,32 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    fastercsv (1.5.4)
+    git (1.2.5)
+    jeweler (1.5.2)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    mechanize (1.0.0)
+      nokogiri (>= 1.2.1)
+    nokogiri (1.4.4)
+    rake (0.8.7)
+    rspec (2.5.0)
+      rspec-core (~> 2.5.0)
+      rspec-expectations (~> 2.5.0)
+      rspec-mocks (~> 2.5.0)
+    rspec-core (2.5.1)
+    rspec-expectations (2.5.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fastercsv
+  jeweler
+  mechanize
+  rake
+  rspec (~> 2)

data/Rakefile

@@ -0,0 +1,25 @@
+task :default do
+  sh "rspec spec/"
+end
+
+task :headers do
+  require File.expand_path('lib/webpagetest_grabber')
+  csv = `curl http://www.webpagetest.org/result/110316_DA_5TFD/110316_DA_5TFD_google.com_page_data.csv`
+  data = PagespeedGrabber.csv_to_array(csv).first.keys
+  puts data.inspect
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = 'pagespeed_grabber'
+    gem.summary = "Grab pagespeed results from external services to use them in e.g. munin."
+    gem.email = "michael@grosser.it"
+    gem.homepage = "http://github.com/grosser/#{gem.name}"
+    gem.authors = ["Michael Grosser"]
+  end
+
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler, or one of its dependencies, is not available. Install it with: gem install jeweler"
+end

data/Readme.md

@@ -0,0 +1,23 @@
+Grab pagespeed results from external services to use them in e.g. munin.<br/>
+Currently only support webpagetest.org, which is not very reliable and sometimes slow.
+
+Not ready for production just a toy project.
+
+    data = PagespeedGrabber.fetch('google.com', :from => 'webpagetest', :timeout => 200)
+
+### Munin
+`munin.rb` can be moved into munin/plugins and renamed to e.g. webpagetest_google.com to aggregate results.
+
+TODO
+====
+ - add other services
+ - make less hacky/use official apis
+ - generate initial AND repeated graph
+ - find a reliable service
+ - parse .har file from e.g. loads.in
+
+Authors
+=======
+[Michael Grosser](http://grosser.it)<br/>
+michael@grosser.it<br/>
+Hereby placed under public domain, do what you want, just do not hold me accountable...

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/lib/pagespeed_grabber.rb

@@ -0,0 +1,46 @@
+require 'rubygems'
+require 'mechanize'
+require 'faster_csv'
+require 'timeout'
+
+module PagespeedGrabber
+  URL = 'http://www.webpagetest.org'
+  VERSION = File.read( File.join(File.dirname(__FILE__),'..','VERSION') ).strip
+  DEFAULT_TIMEOUT = 100
+  TIME_BETWEEN_TRIES = 10
+  CSV_LINK = 'Raw page data'
+  HEADERS = ["Load Time (ms)", "Other Responses (Doc)", "Connections", "Minify Savings", "Experimental", "IP Address", "ETag Score", "GZIP Score", "DNS Lookups", "Event Name", "Not Found", "Segments Transmitted", "Keep-Alive Score", "Time", "Not Modified", "Cookie Score", "Measurement Type", "Gzip Savings", "Time to Base Page Complete (ms)", "One CDN Score", "OK Responses", "Error Code", "unused", "Requests", "Other Responses", "Includes Object Data", "Minify Total Bytes", "Flagged Requests", "AFT (ms)", "Packet Loss (out)", "Compression Score", "URL", "OK Responses (Doc)", "Combine Score", "Base Page Result", "Doc Complete Time (ms)", "Redirects (Doc)", "Pagetest Build", "Minify Score", "Time to Start Render (ms)", "Cache Score", "Base Page Redirects", "Bytes Out (Doc)", "Descriptor", "Dialer ID", "Connection Type", "Activity Time(ms)", "Time to First Byte (ms)", "Date", "Not Found (Doc)", "Redirects", "Not Modified (Doc)", "Event GUID", "Event URL", "Requests (Doc)", "Bytes In (Doc)", "Time to DOM Element (ms)", "Static CDN Score", "Optimization Checked", "Image Savings", "Connections (Doc)", "Flagged Connections", "Max Simultaneous Flagged Connections", "Cached", "Gzip Total Bytes", "Bytes Out", "Bytes In", "DNS Lookups (Doc)", "Segments Retransmitted", "DOCTYPE Score", "Image Total Bytes", "Host", "Lab ID"]
+
+  def self.fetch(test_url, options={})
+    Timeout.timeout(options[:timeout]||DEFAULT_TIMEOUT) do
+      csv_to_array(download_csv(test_url))
+    end
+  end
+
+  def self.download_csv(test_url)
+    agent = Mechanize.new
+    page = agent.get(URL)
+    form = page.forms.first
+    form.field_with('url').value = test_url
+    page = form.submit
+
+    url = page.uri.to_s
+    link = nil
+
+    loop do
+      sleep TIME_BETWEEN_TRIES
+      page = agent.get(url)
+      break if link = page.link_with(:text => CSV_LINK)
+    end
+
+    agent.get(URL + link.href).body
+  end
+
+  def self.csv_to_array(csv)
+    data = []
+    FasterCSV.parse(csv, :headers=>true) do |row|
+      data << row.to_hash
+    end
+    data
+  end
+end

data/munin.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'pagespeed_grabber'
+
+test_url = File.dirname(__FILE__).split('_',2).last
+
+def clean_name(name)
+  name.gsub(/[ \(\)]/,'_').downcase
+end
+
+if ARGV[0].to_s == 'config'
+  puts "graph_title #{test_url} Webpagetest\n";
+  puts "graph_vlabel number\n";
+  puts "graph_scale no\n";
+  puts "graph_category other\n";
+
+  PagespeedGrabber::HEADERS.each do |header|
+    puts "#{clean_name(header)}.label #{header}\n"
+  end
+else
+  initial, repeated = PagespeedGrabber.fetch(test_url)
+  initial.each do |header, value|
+    puts "#{clean_name(header)}.value #{value}"
+  end
+end

data/pagespeed_grabber.gemspec

@@ -0,0 +1,48 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{pagespeed_grabber}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Michael Grosser"]
+  s.date = %q{2011-03-17}
+  s.email = %q{michael@grosser.it}
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "Rakefile",
+    "Readme.md",
+    "VERSION",
+    "lib/pagespeed_grabber.rb",
+    "munin.rb",
+    "pagespeed_grabber.gemspec",
+    "spec/pagespeed_grabber_spec.rb"
+  ]
+  s.homepage = %q{http://github.com/grosser/pagespeed_grabber}
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.4.2}
+  s.summary = %q{Grab pagespeed results from external services to use them in e.g. munin.}
+  s.test_files = [
+    "spec/pagespeed_grabber_spec.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<mechanize>, [">= 0"])
+      s.add_runtime_dependency(%q<fastercsv>, [">= 0"])
+    else
+      s.add_dependency(%q<mechanize>, [">= 0"])
+      s.add_dependency(%q<fastercsv>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<mechanize>, [">= 0"])
+    s.add_dependency(%q<fastercsv>, [">= 0"])
+  end
+end
+

data/spec/pagespeed_grabber_spec.rb

@@ -0,0 +1,8 @@
+require File.expand_path('lib/pagespeed_grabber')
+
+describe PagespeedGrabber do
+  it "can fetch results for an url" do
+    initial, repeated = PagespeedGrabber.fetch('google.com')
+    initial['Load Time (ms)'].to_i.should > repeated['Load Time (ms)'].to_i
+  end
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification 
+name: pagespeed_grabber
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Michael Grosser
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-17 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  requirement: *id001
+  name: mechanize
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  requirement: *id002
+  name: fastercsv
+  prerelease: false
+description: 
+email: michael@grosser.it
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Gemfile
+- Gemfile.lock
+- Rakefile
+- Readme.md
+- VERSION
+- lib/pagespeed_grabber.rb
+- munin.rb
+- pagespeed_grabber.gemspec
+- spec/pagespeed_grabber_spec.rb
+has_rdoc: true
+homepage: http://github.com/grosser/pagespeed_grabber
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.2
+signing_key: 
+specification_version: 3
+summary: Grab pagespeed results from external services to use them in e.g. munin.
+test_files: 
+- spec/pagespeed_grabber_spec.rb