google_ajax_crawler 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    M2QxYzU1Yjc3YmEzYzRmYjg5MTdlMWViN2JiYWUyNjdiYWRmZmQ4Yg==
+  data.tar.gz: !binary |-
+    MDBmOTllZDkzYzg2NjE1ZmVjY2RiYzE4NzA1YjVhNzA4ZTNiMjE4ZA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    OTg5Y2E4MGZjZTRlOWU0ZmRkNDY3OGRmMTMwYzkzZTc3MzIyNDViMTdiMGQy
+    ZmQ5Yjc1ZGM2ZDUwYTk4YjliYzYwYzg1MmY0MzY0ODliNGI5MWZlMTlhYjhm
+    Njg1MTI2MzAzOTY1Mzg2MGYyMmExNDM5YmI4Y2ZkYzBkYmU5MjI=
+  data.tar.gz: !binary |-
+    ZGYzZDVlMmZkOTY2YTU5MWZhNDRlNWQ5MzllMmFmMjVlMjBhYjk2MjM0NDZi
+    YmFkMzE4MDBlMTliYTUzOWE4MWZkNjRmNjA3NTZmOTg5NWE2M2RjNzRmMDA4
+    ZGYyOTc5ZjI2N2NjMTFhZTA4NTdjMDVjZGU2MzIxNTRlNDlkM2M=

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,38 @@
+.rvmrc
+tmp
+# rcov generated
+coverage
+coverage.data
+
+# rdoc generated
+rdoc
+
+# yard generated
+doc
+.yardoc
+
+# bundler
+.bundle
+
+# jeweler generated
+pkg
+
+# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
+#
+# * Create a file at ~/.gitignore
+# * Include files you want ignored
+# * Run: git config --global core.excludesfile ~/.gitignore
+#
+# After doing this, these files will be ignored in all your git projects,
+# saving you from having to 'pollute' every project you touch with them
+#
+# Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
+#
+# For MacOS:
+#
+.DS_Store
+
+# For TextMate
+*.tmproj
+tmtags
+

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/Gemfile

@@ -0,0 +1,11 @@
+source "https://rubygems.org"
+
+gem 'capybara-webkit', '>= 0.10.0'
+gem 'rack'
+
+group :development, :test do
+  gem "simplecov"
+	gem 'rake'
+  gem 'rspec'
+  gem 'faraday'
+end

data/Gemfile.lock

@@ -0,0 +1,60 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    capybara (2.0.2)
+      mime-types (>= 1.16)
+      nokogiri (>= 1.3.3)
+      rack (>= 1.0.0)
+      rack-test (>= 0.5.4)
+      selenium-webdriver (~> 2.0)
+      xpath (~> 1.0.0)
+    capybara-webkit (0.14.2)
+      capybara (~> 2.0, >= 2.0.2)
+      json
+    childprocess (0.3.9)
+      ffi (~> 1.0, >= 1.0.11)
+    diff-lcs (1.2.1)
+    faraday (0.8.6)
+      multipart-post (~> 1.1)
+    ffi (1.4.0)
+    json (1.7.7)
+    mime-types (1.21)
+    multi_json (1.6.1)
+    multipart-post (1.2.0)
+    nokogiri (1.5.6)
+    rack (1.5.2)
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rake (10.0.3)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.0)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.0)
+    rubyzip (0.9.9)
+    selenium-webdriver (2.31.0)
+      childprocess (>= 0.2.5)
+      multi_json (~> 1.0)
+      rubyzip
+      websocket (~> 1.0.4)
+    simplecov (0.7.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.7.1)
+    simplecov-html (0.7.1)
+    websocket (1.0.7)
+    xpath (1.0.0)
+      nokogiri (~> 1.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  capybara-webkit (>= 0.10.0)
+  faraday
+  rack
+  rake
+  rspec
+  simplecov

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2013 Ben Kitzelman
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+# Google Ajax Crawler
+
+Rack Middleware adhering to the Google Ajax Crawling Scheme, using a headless browser to render JS heavy pages and serve a dom snapshot of the rendered state to a requesting search engine.
+
+Details of the scheme can be found at: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+
+## Using
+
+install
+
+``` ruby
+gem install google-ajax-crawler
+```
+
+In your config.ru
+
+``` ruby
+require 'google_ajax_crawler'
+
+use GoogleAjaxCrawler::Crawler do |config|
+  config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('document.getElementById("loading") == null') }
+end
+
+app = lambda {|env| [200, {'Content-Type' => 'text/plain'}, "b" ] }
+run app
+
+```
+
+## Configuration Options
+
+### page_loaded_test
+
+Tell the crawler when your page has finished loading / rendering. As determining when a page has completed rendering can depend on a number of qualitative factors (i.e. all ajax requests have responses, certain content has been displayed, or even when there are no loaders / spinners visible on the page), the page loaded test allows you to specify when the crawler should decide that your page has finished loading / rendering and to return a snapshot of the rendered dom at that time.
+
+The current crawler driver is passed to the lambda to allow querying of the current page's dom state.
+
+### timeout
+
+The max time the crawler should wait before returning a response
+
+### driver
+
+The configured google ajax crawler driver used to query the current page state. Presently there is only one driver (now taking pull requests!); CapybaraWebkit
+
+### poll_interval
+
+How often (in seconds) to test the page state with the configured page_loaded_test
+
+### response_headers
+
+What response headers shoudl be returned with the dom snapshot. Default headers specify the content-type text/html
+
+## License
+
+All free - Use, modify, fork to your hearts content...
+See LICENSE.txt for further details.
+

data/Rakefile

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'

data/google_ajax_crawler.gemspec

@@ -0,0 +1,15 @@
+require './lib/google_ajax_crawler'
+Gem::Specification.new do |s|
+  s.name        = 'google_ajax_crawler'
+  s.version     = GoogleAjaxCrawler.version
+  s.summary     = 'Rack Middleware adhering to the Google Ajax Crawling Scheme ensuring your JS rendered page states (i.e. BackboneJS routes) can be crawled and indexed by search engines.'
+  s.description = 'Rack Middleware adhering to the Google Ajax Crawling Scheme, using a headless browser to render JS heavy pages and serve a dom snapshot of the rendered state to a requesting search engine.'
+  s.authors     = ['Ben Kitzelman']
+  s.email       = ['benkitzelman@gmail.com']
+  s.homepage    = 'http://github.com/benkitzelman/google-ajax-crawler'
+  s.files       = `git ls-files`.strip.split("\n")
+  s.executables = []
+
+  s.add_dependency 'capybara-webkit', '>= 0.10.0'
+  s.add_dependency 'rack'
+end

data/lib/google_ajax_crawler.rb

@@ -0,0 +1,15 @@
+require 'rack/utils'
+require 'uri'
+
+base_path = './lib/google_ajax_crawler'
+
+require "#{base_path}/drivers/driver"
+[base_path, "#{base_path}/drivers"].each do |folder|
+  Dir["#{folder}/*.rb"].each {|file| require file }
+end
+
+module GoogleAjaxCrawler
+  def self.version
+    "0.1.0"
+  end
+end

data/lib/google_ajax_crawler/crawler.rb

@@ -0,0 +1,56 @@
+
+module GoogleAjaxCrawler
+  class Crawler
+
+    class << self
+      def options
+        configure if @options.nil?
+        @options
+      end
+
+      def configure(&block)
+        @options = Options.new(self, &block)
+      end
+    end
+
+    def initialize(app = nil, &block)
+      @app = app
+      self.class.configure &block
+    end
+
+    def options
+      self.class.options
+    end
+
+    def call(env)
+      request = Rack::Request.new(env)
+      if is_search_engine?(request)
+        serve_crawlable_content_for request
+      else
+        @app.call(env)
+      end
+    end
+
+    protected
+
+    def is_search_engine?(request)
+      request.params.include? options.requested_route_key
+    end
+
+    def as_uri_with_fragment(url)
+      uri    = URI.parse(url)
+      params = Rack::Utils.parse_query(uri.query).merge(search_engine: true)
+      uri.fragment = params.delete options.requested_route_key
+      uri.query    = Rack::Utils.build_query params
+      uri
+    end
+
+    def serve_crawlable_content_for(request)
+      puts ' -- GOOGLE Ajax Web Crawler Request --'
+      html = GoogleAjaxCrawler::Page.read as_uri_with_fragment(request.url), options
+
+      [200, options.response_headers, [html]]
+    end
+
+  end
+end

data/lib/google_ajax_crawler/drivers/capybara_webkit.rb

@@ -0,0 +1,28 @@
+require "capybara"
+require "capybara/dsl"
+require "capybara-webkit"
+
+module GoogleAjaxCrawler
+  module Drivers
+    class CapybaraWebkit < Driver
+      include Capybara::DSL
+
+      def initialize *args
+        super *args
+        configure
+      end
+
+      def default_page_loaded_test
+        (page.evaluate_script('$.active') == 0)
+      end
+
+      protected
+
+      def configure
+        Capybara.run_server     = false
+        Capybara.current_driver = :webkit
+        Capybara.default_wait_time = options.timeout
+      end
+    end
+  end
+end

data/lib/google_ajax_crawler/drivers/driver.rb

@@ -0,0 +1,50 @@
+module GoogleAjaxCrawler
+  module Drivers
+    class Driver
+      attr_reader :options
+
+      def initialize(options)
+        @options = options
+      end
+
+      def visit(url)
+        raise "Driver Not Specified"
+      end
+
+      def default_page_loaded_test
+        raise "Driver Not Specified"
+      end
+
+      def html
+        raise "Driver Not Specified"
+      end
+
+      def get_content(uri)
+        puts "requesting: #{uri}"
+        visit uri.to_s
+
+        wait_until_page_is_fully_loaded
+        html
+      end
+
+      def is_page_loaded?
+        if options.page_loaded_test.nil?
+          default_page_loaded_test
+        else
+          options.page_loaded_test.call(self)
+        end
+      end
+
+      def wait_until_page_is_fully_loaded
+        begin
+          while !is_page_loaded?
+            sleep options.poll_interval
+          end
+        rescue
+          #...squelch
+          puts "Timeout: #{$!}"
+        end
+      end
+    end
+  end
+end

data/lib/google_ajax_crawler/options.rb

@@ -0,0 +1,22 @@
+module GoogleAjaxCrawler
+  class Options
+    attr_accessor :driver, :timeout, :requested_route_key, :page_loaded_test, :poll_interval, :response_headers
+
+    def initialize(app, &block)
+      @driver  = Drivers::CapybaraWebkit.new(self)
+      @timeout = 30
+      @requested_route_key  = '_escaped_fragment_'
+      @response_headers     = { 'Content-Type' => 'text/html' }
+      @poll_interval        = 0.5
+
+      instance_exec(self, &block) unless block.nil?
+
+      @app = app
+    end
+
+    def driver=(klass)
+      @driver = klass.new(self)
+    end
+
+  end
+end

data/lib/google_ajax_crawler/page.rb

@@ -0,0 +1,18 @@
+module GoogleAjaxCrawler
+  class Page
+    attr_reader :options
+
+    def self.read(uri, options)
+      page = Page.new(options)
+      page.get_page_content(uri)
+    end
+  
+    def initialize(options)
+      @options = options
+    end
+
+  	def get_page_content(uri = URI.parse("http://localhost"))
+      options.driver.get_content uri
+    end
+  end
+end

data/spec/integration/capybara_webkit_spec.rb

@@ -0,0 +1,35 @@
+require './spec/spec_helper'
+
+describe 'CapybaraWebkit driver' do
+  let(:host)           { "http://localhost:#{RackApp.port}/"}
+  let(:browser_route)  { "#{host}#test" }
+  let(:snapshot_route) { "#{host}?_escaped_fragment_=test" }
+
+  before(:all) do
+    RackApp.configure_crawler do |config|
+      config.driver = GoogleAjaxCrawler::Drivers::CapybaraWebkit
+      config.poll_interval    = 0.25
+      config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('document.getElementById("loading") == null') }
+    end
+
+    RackApp.start
+  end
+
+  after(:all) do
+    RackApp.stop
+  end
+
+  describe 'when a browser requests a client side route (i.e.: /#my_route)' do
+    it 'should not serve a snapshot of the dom' do
+      response = Faraday.get browser_route
+      response.body.should_not =~ /Javascript rendering complete for client-side route #test/
+    end
+  end
+
+  describe 'when an ajax crawler requests a snapshot of a client side route' do
+    it 'should serve a snapshot of the dom that includes js rendered components' do
+      response = Faraday.get snapshot_route
+      response.body.should =~ /Javascript rendering complete for client-side route #test/
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'bundler/setup'
+require './lib/google_ajax_crawler'
+require 'faraday'
+
+here = File.dirname __FILE__
+Dir["#{here}/support/*.rb"].each {|file| require file }
+
+class MockDriver < GoogleAjaxCrawler::Drivers::Driver; end
+

data/spec/support/page.html

@@ -0,0 +1,28 @@
+<html>
+<head></head>
+<body>
+  <h1>A Simple State Test</h1>
+  <p>State: <span id='page_state'></span></p>
+  <div class='loading' id='loading'>Loading....</div>
+
+  <script type='text/javascript'>
+
+  var init = function() {
+    var writeHash = function() {
+      document.getElementById('page_state').innerHTML = "Javascript rendering complete for client-side route " + document.location.hash;
+      var loadingMask = document.getElementById('loading');
+      loadingMask.parentNode.removeChild(loadingMask);
+      console.log('done...');
+    };
+
+    setTimeout(writeHash, 500);
+  };
+
+  //
+  // Only execute js if loading the page using an unescaped url
+  //
+  if(/#.*$/.test(document.location.href)) init();
+
+  </script>
+</body>
+</html>

data/spec/support/rack_app.rb

@@ -0,0 +1,87 @@
+require 'rack'
+
+class RackApp
+
+  def app
+    page_content = File.read('./spec/support/page.html')
+    Rack::Builder.new do
+
+      use GoogleAjaxCrawler::Crawler do |c|
+        RackApp.crawler_configuration.call(c)
+      end
+
+      map '/' do
+        run lambda {|env| [200, { 'Content-Type' => 'text/html' }, [page_content]] }
+      end
+    end
+  end
+
+  class << self
+    attr_reader :crawler_configuration
+
+    def app
+      (@app ||= RackApp.new).app
+    end
+
+    def configure_crawler(&block)
+      @crawler_configuration = block
+    end
+
+    def port
+      9999
+    end
+
+    def start
+      setup!
+      pid = Process.fork
+      if pid.nil?
+        Rack::Server.start(:app => app, :Port => port)
+        sleep 1
+      else
+        File.open(pidfile, 'w') { |f| f.write pid }
+        trap("SIGINT") { stop }
+        Process.detach pid
+      end
+    end
+
+    def stop
+      return unless running?
+
+      Process.kill 9, pid
+      File.delete pidfile
+      puts "[Stopped rack app...]"
+    end
+
+    def setup!
+      FileUtils.mkpath(File.dirname(pidfile))
+      FileUtils.mkpath(File.dirname(logfile))
+    end
+
+    def pidfile
+      "tmp/server.pid"
+    end
+
+    def logfile
+      "tmp/server.log"
+    end
+
+    def pid
+      running? ? File.read(pidfile).to_i : 0
+    end
+
+    def running?
+      File.exists?(pidfile)
+    end
+
+    def restart
+      stop if running?
+      start
+    end
+
+    def log_to_file
+      log = File.new RackApp.logfile, "a"
+      STDOUT.reopen log
+      STDERR.reopen log
+    end
+  end
+end

data/spec/unit/crawler_spec.rb

@@ -0,0 +1,54 @@
+require './spec/spec_helper'
+
+describe GoogleAjaxCrawler::Crawler do
+  before(:each) do
+    GoogleAjaxCrawler::Crawler.configure do |config|
+      config.page_loaded_test = lambda{ page.find('.loading', count: 0) }
+    end
+  end
+
+  shared_examples 'a crawler configurer' do |method, *args|
+    it 'and facilitate the setting of crawler options' do
+      GoogleAjaxCrawler::Crawler.send(method, *args) do |config|
+        config.timeout = 10
+        config.driver  = MockDriver
+      end
+    
+      GoogleAjaxCrawler::Crawler.options.timeout.should == 10
+      GoogleAjaxCrawler::Crawler.options.driver.should be_instance_of(MockDriver)
+    end
+  end
+
+  context 'configure' do
+    it_should_behave_like 'a crawler configurer', :configure
+  end    
+
+  context 'initialize' do
+    it_should_behave_like 'a crawler configurer', :new, nil
+  end
+
+  context 'call' do
+    let(:app) { double(:app) }
+    let(:request) { {
+      'HTTP_METHOD'     => 'GET',
+      'HTTP_HOST'       => 'test.com',
+      'PATH_INFO'       => '/test',
+      'QUERY_STRING'    => 'some_key=some_val',
+      'rack.url_scheme' => 'http',
+      "rack.input"      => :blah
+    } }
+    let(:search_engine_request) { request.merge('QUERY_STRING' => '_escaped_fragment_=test') }
+    let(:crawler) { GoogleAjaxCrawler::Crawler.new app }
+    
+    it 'should delegate non snapshot requests to the configured app' do
+      app.should_receive(:call).with request
+      crawler.call request
+    end
+
+    it 'should generate a rendered snapshot on search engine requests' do
+      GoogleAjaxCrawler::Page.stub(:read).and_return :wibble
+      response = crawler.call search_engine_request
+      response.should == [200, GoogleAjaxCrawler::Crawler.options.response_headers, [:wibble]]
+    end
+  end
+end

data/spec/unit/options_spec.rb

@@ -0,0 +1,35 @@
+require './spec/spec_helper'
+describe GoogleAjaxCrawler::Options do
+  context 'initialize' do
+    let(:app) { Class.new }
+
+    it 'should set default values' do
+      options = GoogleAjaxCrawler::Options.new(app)
+
+      options.timeout.should              == 30
+      options.requested_route_key.should  == '_escaped_fragment_'
+      options.response_headers.should     == { 'Content-Type' => 'text/html' }
+      options.poll_interval.should        == 0.5
+      options.driver.should be_a(GoogleAjaxCrawler::Drivers::CapybaraWebkit)
+      options.page_loaded_test.should be_nil
+    end
+
+    it 'should allow default overrides within block scope' do
+      options = GoogleAjaxCrawler::Options.new(app) do |config|
+        config.requested_route_key  = :some_other_key
+        config.page_loaded_test     = :some_other_page_loaded_test
+        config.poll_interval        = 7000
+        config.response_headers     = :some_other_headers
+        config.timeout              = 20
+        config.driver               = MockDriver
+      end
+
+      options.requested_route_key.should  == :some_other_key
+      options.page_loaded_test.should  == :some_other_page_loaded_test
+      options.poll_interval.should        == 7000
+      options.response_headers.should     == :some_other_headers
+      options.timeout.should              == 20
+      options.driver.should be_instance_of(MockDriver)
+    end
+  end
+end

data/spec/unit/page_spec.rb

@@ -0,0 +1,14 @@
+require './spec/spec_helper'
+describe GoogleAjaxCrawler::Page do
+  context 'read' do
+    let(:uri)     { URI.parse('http://www.test.com') }
+    let(:options) { double(:options) }
+
+    it 'should ask the driver to fetch content from a given uri' do
+      options.stub_chain(:driver, :get_content).with(uri).and_return :wibble
+      content = GoogleAjaxCrawler::Page.read(uri, options)
+      content.should == :wibble
+    end
+
+  end
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: google_ajax_crawler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ben Kitzelman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: capybara-webkit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Rack Middleware adhering to the Google Ajax Crawling Scheme, using a
+  headless browser to render JS heavy pages and serve a dom snapshot of the rendered
+  state to a requesting search engine.
+email:
+- benkitzelman@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .document
+- .gitignore
+- .rspec
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- google_ajax_crawler.gemspec
+- lib/google_ajax_crawler.rb
+- lib/google_ajax_crawler/crawler.rb
+- lib/google_ajax_crawler/drivers/capybara_webkit.rb
+- lib/google_ajax_crawler/drivers/driver.rb
+- lib/google_ajax_crawler/options.rb
+- lib/google_ajax_crawler/page.rb
+- spec/integration/capybara_webkit_spec.rb
+- spec/spec_helper.rb
+- spec/support/page.html
+- spec/support/rack_app.rb
+- spec/unit/crawler_spec.rb
+- spec/unit/options_spec.rb
+- spec/unit/page_spec.rb
+homepage: http://github.com/benkitzelman/google-ajax-crawler
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Rack Middleware adhering to the Google Ajax Crawling Scheme ensuring your
+  JS rendered page states (i.e. BackboneJS routes) can be crawled and indexed by search
+  engines.
+test_files: []