google_ajax_crawler 0.1.2 → 0.1.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NzMwMTEyOWEwMzg3OWNiMTE1MzU1NzdjNDdkZGE5ZDE1YzNiMDk5NQ==
+    Nzk1MTI4M2Y1ZTgyZmEyYzIyNWVlM2ExYjlhNmQxMTYwYzI3MGU0OQ==
   data.tar.gz: !binary |-
-    MGNkNzYyMDg4ZWZjMmRjMDJhM2JiMmE2NTg0MTkyOWY2MDRmMWYyMg==
+    N2Y0ZDhjZmFiOGJkYTQwNDA4Y2JlYzQyNGY3OGU1OGYwNzM3YThkZA==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    OGEyYTY5MWQwYzhkY2NiNjlmZWFmM2JiMjYwY2ZkNDQ0M2VmMjM2ZTMyNzhl
-    NjFjNmRkYWE2M2JkMjhhYTZjZmIyY2RkMzI2MzQ5NGNjOGQ5NWUyOTdjMDg3
-    OTAzNDE3NDgwYTAyMWM1ZDgzMGM0YWZlYTIwMzBiNTk5M2VjMGE=
+    YjYwY2Q0Y2I4NzQwNmYzYjBkYTc0NDljNDJjY2E2MDJlYjRkZTFjNzg5NjBk
+    ZmFjZjQxMGYzNTgyYjBmNDIzMTQ5ODlkM2U4NDhlMTk1YWQ2NzZjODAyYWQx
+    M2VmNTRjZGI2ODhmYjY4MDNiZTdlNDY1MDQyN2FkMTA0MTJlZWM=
   data.tar.gz: !binary |-
-    MDc1ODY4MTI2YzRhNTNiNmQ2YzcwNjliMjA2NTFmMzczM2ZmYmI0NzQxNzI1
-    NGVlODA0NjQ4YmQ4ZmIwYTNkZGZlYzQzNmMwNDNhYjhmOTgyM2NkMjQ5N2Ew
-    YzRmNTU2ZDRhYTE2MWRkNjk0MTg1OTU2OTM2MzY5ODA5ODIwMDM=
+    MDg0MWM3ZmRiODk1NjM1MTk3MWRmYjFhNDhhOThlOGJlN2FjMTJjZjY0OTNi
+    ZjQwYjAzNWI5MTI3N2Q2MTJkNGZhMjcyN2YwYTU2ZGQ4ZjM0MmVjMmVkZmJi
+    Nzg5MDY0ZDJlYTlhNTRlZmQzODUxYmFiZmIwNWQ2Y2NlNzY3NzM=

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+
+rvm:
+  - "1.9.2"
+  - "1.9.3"
+  - "2.0.0"
+
+env:
+  - RACK_ENV=ci
+
+before_install: 
+  - sh -e /etc/init.d/xvfb start
+  - echo "Started xvfb..."
+  
+script:
+  - DISPLAY=:99.0 bundle exec rake

data/README.md

@@ -1,5 +1,8 @@
 # Google Ajax Crawler
 
+[![Build Status](https://travis-ci.org/benkitzelman/google-ajax-crawler.png)](https://travis-ci.org/benkitzelman/google-ajax-crawler)
+[![Gem Version](https://badge.fury.io/rb/google_ajax_crawler.png)](http://badge.fury.io/rb/google_ajax_crawler)
+
 Rack Middleware adhering to the Google Ajax Crawling Scheme, using a headless browser to render JS heavy pages and serve a dom snapshot of the rendered state to a requesting search engine.
 
 Details of the scheme can be found at: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
@@ -44,6 +47,16 @@ Tell the crawler when your page has finished loading / rendering. As determining
 
 The current crawler driver is passed to the lambda to allow querying of the current page's dom state.
 
+A good pattern is to test your page state in a js function returning a boolean, accessible from the window context.. i.e.
+
+```ruby
+
+use GoogleAjaxCrawler::Crawler do |config|
+  config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('myApp.isPageLoaded()') }
+end
+
+```
+
 ### timeout
 
 The max time the crawler should wait before returning a response

data/Rakefile

@@ -1,12 +1,15 @@
-# encoding: utf-8
-
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
+$:.unshift File.join(File.dirname(__FILE__), 'lib')
+
+require 'bundler/setup'
+require 'rspec/core/rake_task'
+
+desc "Run specifications"
+RSpec::Core::RakeTask.new(:spec)
+
+task :ci do
+  puts 'running tests on CI Server....'
+  system("export DISPLAY=:99.0 && bundle exec rake spec")
+  raise "rake spec failed!" unless $?.exitstatus == 0  
 end
-require 'rake'
+
+task :default => :ci

data/examples/capybara_webkit.ru

@@ -1,11 +1,34 @@
+#
+# to run:
+# $ rackup examples/capybara_webkit.ru -p 3000
+# open browser to http://localhost:3000/#!test
+#
 require 'bundler/setup'
 require './lib/google_ajax_crawler'
 
 use GoogleAjaxCrawler::Crawler do |config|
-  config.driver = GoogleAjaxCrawler::Drivers::CapybaraWebkit
-  config.poll_interval    = 0.25
-  config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('document.getElementById("loading") == null') }
+  config.driver        = GoogleAjaxCrawler::Drivers::CapybaraWebkit
+  config.poll_interval = 0.25
+  config.timeout       = 5
+
+  #
+  # for the demo - the page is considered loaded when the loading mask has been removed from the DOM
+  # this could evaluate something like $.active == 0 to ensure no jquery ajax calls are pending
+  #
+  config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('app.pageHasLoaded()') }
 end
 
-page_content = File.read('./spec/support/page.html')
-run lambda {|env| [200, { 'Content-Type' => 'text/html' }, [page_content]] }
+#
+# a sample page using #! url fragments to seed page state
+#
+app = lambda do |env|
+  page_content = case env['PATH_INFO']
+    when /\/backbone(\/)?/
+      File.read('./spec/fixtures/backbone.html')
+    else
+      File.read('./spec/fixtures/simple_javascript.html')
+  end
+
+  [200, { 'Content-Type' => 'text/html' }, [page_content]]
+end
+run app

data/google_ajax_crawler.gemspec

@@ -1,5 +1,4 @@
-$LOAD_PATH << './lib'
-require 'google_ajax_crawler'
+require './lib/google_ajax_crawler'
 Gem::Specification.new do |s|
   s.name        = 'google_ajax_crawler'
   s.version     = GoogleAjaxCrawler.version
@@ -13,4 +12,4 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'capybara-webkit', '>= 0.10.0'
   s.add_dependency 'rack'
-end
+end

data/lib/google_ajax_crawler.rb

@@ -1,14 +1,19 @@
 module GoogleAjaxCrawler
   class << self
+    def env
+      (ENV['RACK_ENV'] || 'development').to_sym
+    end
+
     def version
-      "0.1.2"
+      "0.1.3"
     end
   end
 end
 
+here = File.dirname(__FILE__)
 require 'uri'
-require 'google_ajax_crawler/crawler'
-require 'google_ajax_crawler/options'
-require 'google_ajax_crawler/page'
-require 'google_ajax_crawler/drivers/driver'
-require 'google_ajax_crawler/drivers/capybara_webkit'
+require "#{here}/google_ajax_crawler/crawler"
+require "#{here}/google_ajax_crawler/options"
+require "#{here}/google_ajax_crawler/page"
+require "#{here}/google_ajax_crawler/drivers/driver"
+require "#{here}/google_ajax_crawler/drivers/capybara_webkit"

data/lib/google_ajax_crawler/crawler.rb

@@ -24,11 +24,9 @@ module GoogleAjaxCrawler
 
     def call(env)
       request = Rack::Request.new(env)
-      if is_search_engine?(request)
-        serve_crawlable_content_for request
-      else
-        @app.call(env)
-      end
+      return serve_crawlable_content_for(request) if is_search_engine?(request)
+      
+      @app.call(env)
     end
 
     protected

data/lib/google_ajax_crawler/drivers/driver.rb

@@ -1,3 +1,4 @@
+require 'timeout'
 module GoogleAjaxCrawler
   module Drivers
     class Driver
@@ -20,10 +21,16 @@ module GoogleAjaxCrawler
       end
 
       def get_content(uri)
-        puts "requesting: #{uri}"
-        visit uri.to_s
-
-        wait_until_page_is_fully_loaded
+        begin
+          puts "::requesting: #{uri}"
+          visit uri.to_s
+          wait_until_page_is_fully_loaded
+        rescue Timeout::Error
+          puts  "-- Page Rendering Timed out: --\n"\
+                "Either your page_loaded_test didn't successfully detect when your page had loaded, \n"\
+                "or your page took longer than #{options.timeout} seconds to load \n"\
+                "-- Returning page snapshot in its present state --"
+        end
         html
       end
 
@@ -31,18 +38,20 @@ module GoogleAjaxCrawler
         if options.page_loaded_test.nil?
           default_page_loaded_test
         else
-          options.page_loaded_test.call(self)
+          options.page_loaded_test.call self
         end
       end
 
       def wait_until_page_is_fully_loaded
-        begin
-          while !is_page_loaded?
-            sleep options.poll_interval
+        Timeout::timeout(options.timeout) do
+          begin
+            while !is_page_loaded?
+              sleep options.poll_interval
+            end
+          rescue
+            #...squelch
+            puts "Exception: #{$!}"
           end
-        rescue
-          #...squelch
-          puts "Timeout: #{$!}"
         end
       end
     end

data/spec/fixtures/backbone.html

@@ -0,0 +1,61 @@
+<html>
+  <head>
+    <script src='http://cdnjs.cloudflare.com/ajax/libs/underscore.js/1.4.4/underscore-min.js'></script>
+    <script src='http://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.0/jquery.min.js'></script>
+    <script src='http://cdnjs.cloudflare.com/ajax/libs/backbone.js/1.0.0/backbone-min.js'></script>
+  </head>
+<body>
+
+  <div id='app'>
+    <div id='loading'>Loading...</div>
+  </div>
+
+  <script type='text/javascript'>
+
+    var View = Backbone.View.extend({
+      className: 'backbone-view',
+      template: _.template(
+        '<h1>Backbone Example</h1><p>This template was rendered by backbone js</p>'
+      ),
+
+      render: function(){
+        var app   = document.getElementById('app');
+        var _this = this;
+        this.el.innerHTML = _this.template();
+        return this;
+      }
+    }); 
+
+    var Router = Backbone.Router.extend({
+      routes: {
+        "*anything": "home"
+      },
+
+      home: function() {
+        el = document.getElementById('app');
+        var view = new View;
+
+        // simulate fetching data / async operation
+        _.delay(function() {
+          el.replaceChild(view.render().el, el.children[0]);
+        }, 1500);
+      }
+    });
+
+    function App() {
+      new Router();
+      Backbone.history.start();
+    };
+
+    App.prototype.pageHasLoaded = function() {
+      return !!document.getElementsByClassName('backbone-view').length;
+    };
+
+
+    $(function() {
+      window.app = new App();
+    });
+
+  </script>
+</body>
+</html>

data/spec/{support/page.html → fixtures/simple_javascript.html}

@@ -7,7 +7,8 @@
 
   <script type='text/javascript'>
 
-  var init = function() {
+  
+  function App() {
     var writeHash = function() {
       document.getElementById('page_state').innerHTML = "Javascript rendering complete for client-side route " + document.location.hash;
       var loadingMask = document.getElementById('loading');
@@ -19,10 +20,14 @@
     setTimeout(writeHash, 500);
   };
 
+  App.prototype.pageHasLoaded = function() {
+    return !document.getElementById('loading');
+  };
+
   //
   // Only execute js if loading the page using an unescaped url
   //
-  if(/#.*$/.test(document.location.href)) init();
+  if(/#.*$/.test(document.location.href)) window.app = new App();
 
   </script>
 </body>

data/spec/integration/capybara_webkit_spec.rb

@@ -9,7 +9,7 @@ describe 'CapybaraWebkit driver' do
     RackApp.configure_crawler do |config|
       config.driver = GoogleAjaxCrawler::Drivers::CapybaraWebkit
       config.poll_interval    = 0.25
-      config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('document.getElementById("loading") == null') }
+      config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('app.pageHasLoaded()') }
     end
 
     RackApp.start

data/spec/spec_helper.rb

@@ -1,3 +1,5 @@
+ENV["RACK_ENV"] ||= "test"
+
 require 'rubygems'
 require 'bundler/setup'
 require './lib/google_ajax_crawler'

data/spec/support/rack_app.rb

@@ -3,7 +3,7 @@ require 'rack'
 class RackApp
 
   def app
-    page_content = File.read('./spec/support/page.html')
+    page_content = File.read('./spec/fixtures/simple_javascript.html')
     Rack::Builder.new do
 
       use GoogleAjaxCrawler::Crawler do |c|

data/spec/unit/drivers/driver_spec.rb

@@ -0,0 +1,19 @@
+require './spec/spec_helper'
+
+describe GoogleAjaxCrawler::Drivers::Driver do
+  let(:options) do 
+    GoogleAjaxCrawler::Options.new(nil) do |o|
+      o.timeout = 0.05
+      o.page_loaded_test = lambda {|d| false }
+    end
+  end
+
+  describe '#wait_until_page_is_fully_loaded' do
+    it 'should raise a Timeout Exception when timeout limit reached' do
+      expect do
+        driver = GoogleAjaxCrawler::Drivers::Driver.new(options)
+        driver.wait_until_page_is_fully_loaded
+      end.to raise_error(Timeout::Error)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google_ajax_crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Ben Kitzelman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-16 00:00:00.000000000 Z
+date: 2013-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara-webkit
@@ -49,6 +49,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .rspec
+- .travis.yml
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -64,11 +65,14 @@ files:
 - lib/google_ajax_crawler/page.rb
 - releases/google_ajax_crawler-0.1.0.gem
 - releases/google_ajax_crawler-0.1.1.gem
+- releases/google_ajax_crawler-0.1.2.gem
+- spec/fixtures/backbone.html
+- spec/fixtures/simple_javascript.html
 - spec/integration/capybara_webkit_spec.rb
 - spec/spec_helper.rb
-- spec/support/page.html
 - spec/support/rack_app.rb
 - spec/unit/crawler_spec.rb
+- spec/unit/drivers/driver_spec.rb
 - spec/unit/options_spec.rb
 - spec/unit/page_spec.rb
 homepage: http://github.com/benkitzelman/google-ajax-crawler