google_ajax_crawler 0.1.3 → 0.2.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    Nzk1MTI4M2Y1ZTgyZmEyYzIyNWVlM2ExYjlhNmQxMTYwYzI3MGU0OQ==
+    YTU0N2ZkNGQ0YzJiMDlkYmYyMTI4YWNkMDY0ZmQ2MmM1MzBkZGNkNQ==
   data.tar.gz: !binary |-
-    N2Y0ZDhjZmFiOGJkYTQwNDA4Y2JlYzQyNGY3OGU1OGYwNzM3YThkZA==
+    ODdkODI5OGI3NGIzYTdhOWNlOTEwMGE5NzRkZjgxYzUzOTY0NzZjZg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    YjYwY2Q0Y2I4NzQwNmYzYjBkYTc0NDljNDJjY2E2MDJlYjRkZTFjNzg5NjBk
-    ZmFjZjQxMGYzNTgyYjBmNDIzMTQ5ODlkM2U4NDhlMTk1YWQ2NzZjODAyYWQx
-    M2VmNTRjZGI2ODhmYjY4MDNiZTdlNDY1MDQyN2FkMTA0MTJlZWM=
+    ZTBlNjdlOWI3NDU2ZDI3OTBkYmU5MDE4MTJlZWIxZDZmODMzYTg1NmUyMjE1
+    OTExMDNjY2UyOWZiZjljOGUwYzhhZTJiZjFmODY2OTE3NmFkYzEyNWZhY2E3
+    M2E1MzlmZTljYjVmMTBmOWQ0ZDIzMzQyMWY1NTA4NWY3MjgwYzE=
   data.tar.gz: !binary |-
-    MDg0MWM3ZmRiODk1NjM1MTk3MWRmYjFhNDhhOThlOGJlN2FjMTJjZjY0OTNi
-    ZjQwYjAzNWI5MTI3N2Q2MTJkNGZhMjcyN2YwYTU2ZGQ4ZjM0MmVjMmVkZmJi
-    Nzg5MDY0ZDJlYTlhNTRlZmQzODUxYmFiZmIwNWQ2Y2NlNzY3NzM=
+    ODc3YzBjYzg0MzYyYTA5NzVkM2M2OWRkYWFjNjNlMDQ2MzI2OTI4NjE5MDc2
+    YTUzNmJkNzFlMTE4ZjJiNmRlYzcyMDNhZTUxN2YwOTQ4Y2U0M2MyZDRjNzU5
+    MGNjZDhlZTllY2NjYzUzODljOWJjYjliOGUzNWM0NThkZTE1NjY=

data/.travis.yml

@@ -10,7 +10,6 @@ env:
 
 before_install: 
   - sh -e /etc/init.d/xvfb start
-  - echo "Started xvfb..."
   
 script:
   - DISPLAY=:99.0 bundle exec rake

data/README.md

@@ -9,7 +9,7 @@ Details of the scheme can be found at: https://developers.google.com/webmasters/
 
 ## Using
 
-install
+### install
 
 ``` ruby
 gem install google_ajax_crawler
@@ -21,57 +21,114 @@ In your config.ru
 require 'google_ajax_crawler'
 
 use GoogleAjaxCrawler::Crawler do |config|
-  config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('document.getElementById("loading") == null') }
+  config.page_loaded_js = "MyApp.isPageLoaded()"
 end
 
-app = lambda {|env| [200, {'Content-Type' => 'text/plain'}, "b" ] }
+app = -> env { [200, {'Content-Type' => 'text/plain'}, "b" ] }
 run app
 
 ```
 
+### rails usage
+
+create in the initializer folder :
+``` ruby
+google_ajax_crawler_middleware.rb
+```
+
+with 
+``` ruby
+if defined?(Rails.configuration) && Rails.configuration.respond_to?(:middleware)
+  require 'google_ajax_crawler'
+  Rails.configuration.middleware.use GoogleAjaxCrawler::Crawler do |config|
+    config.page_loaded_test = -> driver { driver.page.evaluate_script('document.getElementById("loading") == null') }
+  end
+end
+```
+
+#### Important
+
+Concurrent requests must be enabled to allow your site to snapshot itself. If concurrent requests are not allowed, the site will simple hang on a crawler request.
+
+In config/application.rb :
+
+``` ruby
+config.threadsafe!
+```
+
 ## Examples
 
-In the examples folder, each driver has a rackup file, which can be launched:
+In the examples folder, each driver has a rackup file (at the moment only one driver, capybara-webkit, exists), which can be launched:
+
+`rackup examples/capybara_webkit.ru`
 
-`rackup examples/[driver_name].ru`
+Examples for how to use the crawler with Backbone.JS, Angular.JS and plain ol javascript are accesible via:
+ - http://localhost:9292/backbone
+ - http://localhost:9292/angular
+ - http://localhost:9292/
 
-then open a browser to http://localhost:9292/#!test and view source.... This is how a search engine will see your page. *NOTE:* don't look at the markup through a web inspector as it will most likely display dom elements rendered on the fly by js.
+Curl, or open a browser to http://localhost:9292/[framework]#!test and view source.... This is how a search engine will see your page before snapshotting. *NOTE:* don't look at the markup through a web inspector as it will most likely display dom elements rendered on the fly by js.
 
-Change the url to http://localhost:9292/?_escaped_fragment_=test , and then again view source to see how the DOM state has been captured
+Change the url to http://localhost:9292/[framework]?_escaped_fragment_=test , and then again curl or view source to see how the DOM state has been captured
 
 ## Configuration Options
 
-### page_loaded_test
+### Page Loaded Tests
 
-Tell the crawler when your page has finished loading / rendering. As determining when a page has completed rendering can depend on a number of qualitative factors (i.e. all ajax requests have responses, certain content has been displayed, or even when there are no loaders / spinners visible on the page), the page loaded test allows you to specify when the crawler should decide that your page has finished loading / rendering and to return a snapshot of the rendered dom at that time.
+As determining when a page has completed rendering can depend on a number of qualitative factors (i.e. all ajax requests have responses, certain content has been displayed, or even when there are no loaders / spinners visible on the page), you can specify one of two ways to tell the crawler that your page has finished loading / rendering and to return a snapshot of the rendered dom at that time.
 
-The current crawler driver is passed to the lambda to allow querying of the current page's dom state.
+#### page_loaded_js (client side test)
 
-A good pattern is to test your page state in a js function returning a boolean, accessible from the window context.. i.e.
+Tell the crawler the client side javascript function (returning true/false) you have created, that determines when your page has finished loading / rendering.
 
 ```ruby
 
 use GoogleAjaxCrawler::Crawler do |config|
-  config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('myApp.isPageLoaded()') }
+  config.page_loaded_js = "MyApp.isPageLoaded()"
+end
+
+```
+
+#### page_loaded_test (server side test)
+
+A server side test determining when your page has finished loading / rendering.
+The configured crawler driver is passed to the lambda to allow querying of the current page's dom state from the server side.
+
+```ruby
+
+use GoogleAjaxCrawler::Crawler do |config|
+  config.page_loaded_test = -> driver { driver.page.has_css?('.loading') == false }
 end
 
 ```
 
 ### timeout
 
-The max time the crawler should wait before returning a response
+The max time (in seconds) the crawler should wait before returning a response. After the timeout has been reached,
+a snapshot of the DOM in its current state is returned. Defaults to 30 seconds.
 
 ### driver
 
-The configured google ajax crawler driver used to query the current page state. Presently there is only one driver (now taking pull requests!); CapybaraWebkit
+The configured google ajax crawler driver used to query the current page state. Defaults to capybara_webkit.
 
 ### poll_interval
 
-How often (in seconds) to test the page state with the configured page_loaded_test
+How often (in seconds) to test the page state with the configured page_loaded_test. Defaults to 0.5 seconds.
 
 ### response_headers
 
-What response headers shoudl be returned with the dom snapshot. Default headers specify the content-type text/html
+What response headers shoudl be returned with the dom snapshot. Default headers specify the content-type text/html.
+
+### requested_route_key
+
+The parameter name used by a search bot to idenitfy which client side route to snapshot. Defaults to _escaped_fragment_.
+
+
+
+### Identifing Search Engine Requests
+
+Snapshot requests are passed an additional query string param (?search_engine=true), allowing you to optionally execute client side code.
+This is particularly handy should you have stats tracking code (i.e. Google Analytics), which you don't want executed / included when search engines are trawling your site.
 
 ## License
 

data/examples/capybara_webkit.ru

@@ -2,6 +2,8 @@
 # to run:
 # $ rackup examples/capybara_webkit.ru -p 3000
 # open browser to http://localhost:3000/#!test
+# or http://localhost:3000/backbone/#!test
+# or http://localhost:3000/backbone/#!test
 #
 require 'bundler/setup'
 require './lib/google_ajax_crawler'
@@ -12,19 +14,25 @@ use GoogleAjaxCrawler::Crawler do |config|
   config.timeout       = 5
 
   #
-  # for the demo - the page is considered loaded when the loading mask has been removed from the DOM
-  # this could evaluate something like $.active == 0 to ensure no jquery ajax calls are pending
+  # for the demo - in each example (simple, backbone and angular) there is a page loaded function signaling
+  # when the page has completed loading. If neither page_loaded_js or page_loaded_test has been configured, the crawler will default to
+  # executing $.active == 0 to ensure no jquery ajax calls are pending
   #
-  config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('app.pageHasLoaded()') }
+  config.page_loaded_js = 'app.pageHasLoaded()'
+
+  # alertnative server side test for the simple_javascript example:
+  # config.page_loaded_test = -> driver { !driver.page.has_css?('#loading') }
 end
 
 #
 # a sample page using #! url fragments to seed page state
 #
-app = lambda do |env|
+app = -> env do 
   page_content = case env['PATH_INFO']
     when /\/backbone(\/)?/
       File.read('./spec/fixtures/backbone.html')
+    when /\/angular(\/)?/
+      File.read('./spec/fixtures/angular.html')
     else
       File.read('./spec/fixtures/simple_javascript.html')
   end

data/lib/google_ajax_crawler.rb

@@ -5,7 +5,7 @@ module GoogleAjaxCrawler
     end
 
     def version
-      "0.1.3"
+      "0.2.0"
     end
   end
 end

data/lib/google_ajax_crawler/drivers/driver.rb

@@ -12,6 +12,10 @@ module GoogleAjaxCrawler
         raise "Driver Not Specified"
       end
 
+      def evaluate_script(javascript)
+        raise "Driver Not Specified"
+      end
+
       def default_page_loaded_test
         raise "Driver Not Specified"
       end
@@ -35,19 +39,15 @@ module GoogleAjaxCrawler
       end
 
       def is_page_loaded?
-        if options.page_loaded_test.nil?
-          default_page_loaded_test
-        else
-          options.page_loaded_test.call self
-        end
+        return evaluate_script(options.page_loaded_js) unless options.page_loaded_js.nil?
+        return options.page_loaded_test.call(self) unless options.page_loaded_test.nil?
+        default_page_loaded_test
       end
 
       def wait_until_page_is_fully_loaded
         Timeout::timeout(options.timeout) do
           begin
-            while !is_page_loaded?
-              sleep options.poll_interval
-            end
+            sleep(options.poll_interval) while !is_page_loaded?
           rescue
             #...squelch
             puts "Exception: #{$!}"

data/lib/google_ajax_crawler/options.rb

@@ -1,6 +1,6 @@
 module GoogleAjaxCrawler
   class Options
-    attr_accessor :driver, :timeout, :requested_route_key, :page_loaded_test, :poll_interval, :response_headers
+    attr_accessor :driver, :timeout, :requested_route_key, :page_loaded_test, :page_loaded_js, :poll_interval, :response_headers
 
     def initialize(app, &block)
       @driver  = Drivers::CapybaraWebkit.new(self)

data/spec/fixtures/angular.html

@@ -0,0 +1,32 @@
+<!doctype html>
+<html ng-app>
+  <head>
+    <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.0.6/angular.min.js"></script>
+  </head>
+  <body>
+
+    <div ng-controller="renderTSCtrl">
+      <h1 id='title'>Angular tested route: {{route()}}</h1>
+      <p id='ts'>{{renderTime()}}</p>
+    </div>
+
+    <script type='text/javascript'>
+      function renderTSCtrl ($scope) {
+        $scope.route = function() {
+          return document.location.hash;
+        };
+
+        $scope.renderTime = function() {
+          return "This rendered at " + new Date().toString() +"!";
+        };
+      };
+
+      window.app = {
+        pageHasLoaded: function () {
+          return !!document.getElementById('title').innerText && !!document.getElementById('ts').innerText;
+        }
+      };
+
+    </script>
+  </body>
+</html>

data/spec/integration/capybara_webkit_spec.rb

@@ -5,31 +5,55 @@ describe 'CapybaraWebkit driver' do
   let(:browser_route)  { "#{host}#!test" }
   let(:snapshot_route) { "#{host}?_escaped_fragment_=test" }
 
-  before(:all) do
-    RackApp.configure_crawler do |config|
-      config.driver = GoogleAjaxCrawler::Drivers::CapybaraWebkit
-      config.poll_interval    = 0.25
-      config.page_loaded_test = lambda {|driver| driver.page.evaluate_script('app.pageHasLoaded()') }
+  shared_examples 'google ajax crawler' do
+    describe 'when a browser requests a client side route (i.e.: /#my_route)' do
+      it 'should not serve a snapshot of the dom' do
+        response = Faraday.get browser_route
+        response.body.should_not =~ /Javascript rendering complete for client-side route #!test/
+      end
     end
 
-    RackApp.start
+    describe 'when an ajax crawler requests a snapshot of a client side route' do
+      it 'should serve a snapshot of the dom that includes js rendered components' do
+        response = Faraday.get snapshot_route
+        response.body.should =~ /Javascript rendering complete for client-side route #!test/
+      end
+    end
   end
 
-  after(:all) do
-    RackApp.stop
-  end
+  describe 'with page_loaded_test' do
+    before(:all) do
+      RackApp.configure_crawler do |config|
+        config.driver = GoogleAjaxCrawler::Drivers::CapybaraWebkit
+        config.poll_interval    = 0.25
+        config.page_loaded_test = -> driver { driver.page.evaluate_script('app.pageHasLoaded()') }
+      end
 
-  describe 'when a browser requests a client side route (i.e.: /#my_route)' do
-    it 'should not serve a snapshot of the dom' do
-      response = Faraday.get browser_route
-      response.body.should_not =~ /Javascript rendering complete for client-side route #!test/
+      RackApp.start
     end
+
+    after(:all) do
+      RackApp.stop
+    end
+
+    it_should_behave_like 'google ajax crawler'
   end
 
-  describe 'when an ajax crawler requests a snapshot of a client side route' do
-    it 'should serve a snapshot of the dom that includes js rendered components' do
-      response = Faraday.get snapshot_route
-      response.body.should =~ /Javascript rendering complete for client-side route #!test/
+  describe 'with page_loaded_js' do
+    before(:all) do
+      RackApp.configure_crawler do |config|
+        config.driver = GoogleAjaxCrawler::Drivers::CapybaraWebkit
+        config.poll_interval  = 0.25
+        config.page_loaded_js = 'app.pageHasLoaded()'
+      end
+
+      RackApp.start
+    end
+
+    after(:all) do
+      RackApp.stop
     end
+
+    it_should_behave_like 'google ajax crawler'
   end
 end

data/spec/unit/crawler_spec.rb

@@ -1,12 +1,6 @@
 require './spec/spec_helper'
 
 describe GoogleAjaxCrawler::Crawler do
-  before(:each) do
-    GoogleAjaxCrawler::Crawler.configure do |config|
-      config.page_loaded_test = lambda{ page.find('.loading', count: 0) }
-    end
-  end
-
   shared_examples 'a crawler configurer' do |method, *args|
     it 'and facilitate the setting of crawler options' do
       GoogleAjaxCrawler::Crawler.send(method, *args) do |config|

data/spec/unit/drivers/driver_spec.rb

@@ -1,17 +1,59 @@
 require './spec/spec_helper'
 
 describe GoogleAjaxCrawler::Drivers::Driver do
-  let(:options) do 
-    GoogleAjaxCrawler::Options.new(nil) do |o|
-      o.timeout = 0.05
-      o.page_loaded_test = lambda {|d| false }
+  let(:options) { GoogleAjaxCrawler::Options.new(nil) { |o| o.timeout = 0.01 } }
+  let(:driver)  { GoogleAjaxCrawler::Drivers::Driver.new(options) }
+
+  describe '#mandatory overrides' do
+    shared_examples 'an enforced override method' do |method, *args|
+      it 'should throw an exception if not overridden' do
+        expect { driver.send(method, *args) }.to raise_error(RuntimeError, "Driver Not Specified")
+      end
+    end
+
+    it_should_behave_like 'an enforced override method', :visit, 'http://test.com'
+    it_should_behave_like 'an enforced override method', :evaluate_script, 'myApp.isPageLoaded()'
+    it_should_behave_like 'an enforced override method', :default_page_loaded_test
+    it_should_behave_like 'an enforced override method', :html
+  end
+
+  describe '#is_page_loaded?' do
+
+    describe 'when page_loaded_test optioned' do
+      it 'should be called' do
+        driver.options.page_loaded_test = double
+        driver.options.page_loaded_test.should_receive(:call).with(driver)
+        driver.is_page_loaded?
+      end
+    end
+
+    describe 'when page_loaded_js optioned' do
+      it 'should call evaluate_script with the page_loaded_js' do
+        driver.options.page_loaded_js = 'MyApp.isPageLoaded()'
+        driver.stub :evaluate_script
+        driver.should_receive(:evaluate_script).with('MyApp.isPageLoaded()').once
+        driver.is_page_loaded?
+      end
+    end
+
+    describe 'when no loaded tests optioned' do
+      it 'should execute the default_page_loaded_test' do
+        driver.options.page_loaded_test = driver.options.page_loaded_js = nil
+        driver.stub :default_page_loaded_test
+        driver.should_receive(:default_page_loaded_test).once
+        driver.is_page_loaded?
+      end
     end
   end
 
   describe '#wait_until_page_is_fully_loaded' do
+    before do
+      driver.options.page_loaded_test = double
+      driver.options.page_loaded_test.should_receive(:call).with(driver)
+    end
+
     it 'should raise a Timeout Exception when timeout limit reached' do
       expect do
-        driver = GoogleAjaxCrawler::Drivers::Driver.new(options)
         driver.wait_until_page_is_fully_loaded
       end.to raise_error(Timeout::Error)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google_ajax_crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Ben Kitzelman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-16 00:00:00.000000000 Z
+date: 2013-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara-webkit
@@ -66,6 +66,8 @@ files:
 - releases/google_ajax_crawler-0.1.0.gem
 - releases/google_ajax_crawler-0.1.1.gem
 - releases/google_ajax_crawler-0.1.2.gem
+- releases/google_ajax_crawler-0.1.3.gem
+- spec/fixtures/angular.html
 - spec/fixtures/backbone.html
 - spec/fixtures/simple_javascript.html
 - spec/integration/capybara_webkit_spec.rb
@@ -94,7 +96,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.0.5
 signing_key: 
 specification_version: 4
 summary: Rack Middleware adhering to the Google Ajax Crawling Scheme ensuring your