prerender_rails 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9131d6f39c2dc688b5255b910007e5a434fe85a5
-  data.tar.gz: fd614761588694380e2abbb6ec3c6a32c64d42f8
+  metadata.gz: 4e710c94d4517957a7198a878b74edb3d2ce599a
+  data.tar.gz: 3d61079a206b75b6ebf47d2acd64ed1774fa1b25
 SHA512:
-  metadata.gz: cd0a338c2026fa5c6abd80bc97410619df877ff94b2dc44e283b6f7c6119959aebb9b8dd307deec5f42f9a88d019ca3977c74f748175071ae3a28c771b2a1d15
-  data.tar.gz: 4fd993cf3da044b7a2fa60da6e09be5d83cd51cdcceeb855c3f47edf58564d41d0e7b7e63a6ce58f08c3baa9daca80b84ea53ecf536901e68cbef6370d163d37
+  metadata.gz: e2f1fd70277c97905a06b2e674ca4e882aea82700c8abdc65eb65e6b4f2a016392f1bb97482bcd8e7fc87269f26ef51ced69b8cdfb2e937abff8b85c209c9abe
+  data.tar.gz: d5f453676bf4373c2c6634713684ccd938ca1761cea76518ec29aff5b2f3a5ce5567fc8083befa2df2d6afe31dd8ba78c7f6191f1319df06246d548d415373b4

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.0.0

data/README.md

@@ -1,46 +1,68 @@
-Prerender Rails
+Prerender Rails [![Build Status](https://travis-ci.org/collectiveip/prerender_rails.png)](https://travis-ci.org/collectiveip/prerender_rails)
 =========================== 
 
-This gem installs middleware that prerenders a javascript-rendered page and sends the HTML to a search engine crawler for SEO. It checks the agent string on the requesting browser for a crawler, and continues on to your normal routes if the requester is not a crawler.
+Are you using backbone, angular, emberjs, etc, but you're unsure about the SEO implications?
 
-## How it works
-* Check to make sure the request is from a crawler and we aren't requesting a resource (js, css, etc...)
-* Make a `GET` request to the [prerender service](https://github.com/collectiveip/prerender) for the page's prerendered HTML
-* Return that HTML to the crawler
-
-## Installation
+Use this gem to install rails middleware that prerenders a javascript-rendered page and returns the HTML to the search engine crawler for SEO.
 
 Add this line to your application's Gemfile:
 
     gem 'prerender_rails'
 
-And then execute:
+And in `config/environment/production.rb`, add this line:
+
+```ruby
+	config.middleware.use Rack::Prerender
+```
 
-    $ bundle
+## How it works
+1. Check to make sure we should show a prerendered page
+	1. Check if the request is from a crawler (agent string)
+	2. Check to make sure we aren't requesting a resource (js, css, etc...)
+	3. (optional) Check to make sure the url is in the whitelist
+	4. (optional) Check to make sure the url isn't in the blacklist
+2. Make a `GET` request to the [prerender service](https://github.com/collectiveip/prerender)(phantomjs server) for the page's prerendered HTML
+3. Return that HTML to the crawler
 
-Or install it yourself as:
+## Customization
 
-    $ gem install prerender_rails
+### Whitelist
 
-## Usage
+Whitelist a single url path or multiple url paths. If a whitelist is supplied, only url's containing a whitelist path will be prerendered.
+```ruby
+config.middleware.use Rack::Prerender, whitelist: '/search/tto'
+```
+```ruby
+config.middleware.use Rack::Prerender, whitelist: ['/search', '/profile']
+```
 
-In `config/environment/production.rb`, add this line:
+### Blacklist
 
+Blacklist a single url path or multiple url paths. If a blacklist is supplied, all url's will be prerendered except ones containing a blacklist path.
 ```ruby
-	config.middleware.use Rack::Prerender
+config.middleware.use Rack::Prerender, blacklist: '/search/tto'
 ```
+```ruby
+config.middleware.use Rack::Prerender, blacklist: ['/search', '/profile']
+```
+
+### Using your own prerender service
+
+If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://prerender.herokuapp.com`
 
-If you've deployed the prerender service on your own, set the `PRERENDER_URL` environment variable so that this gem points there instead. Otherwise, it will default to the service already deployed at `http://prerender.herokuapp.com`
+	$ export PRERENDER_SERVICE_URL=<new url>
 
-	$ export PRERENDER_URL=<new url>
+## Testing
 
-## Contributing
+If you want to make sure your pages are rendering correctly:
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+1. Open the Developer Tools in Chrome (Cmd + Atl + J)
+2. Click the Settings gear in the bottom right corner.
+3. Click "Overrides" on the left side of the settings panel.
+4. Check the "User Agent" checkbox.
+6. Choose "Other..." from the User Agent dropdown.
+7. Type `googlebot` into the input box.
+8. Refresh the page (make sure to keep the developer tools open).
 
 ## License
 

data/Rakefile

@@ -1 +1,12 @@
+#!/usr/bin/env rake
 require "bundler/gem_tasks"
+ 
+require 'rake/testtask'
+ 
+Rake::TestTask.new do |t|
+  t.libs << 'lib/prerender_rails'
+  t.test_files = FileList['test/lib/prerender_rails.rb']
+  t.verbose = true
+end
+ 
+task :default => :test

data/lib/prerender_rails.rb

@@ -1,10 +1,30 @@
-require "rack/version"
-
 module Rack
   class Prerender
     require 'net/http'
 
-    def initialize(app)
+    def initialize(app, options={})
+      @crawler_user_agents = [
+        'googlebot',
+        'yahoo',
+        'bingbot',
+        'baiduspider'
+      ]
+
+      @extensions_to_ignore = [
+        '.js',
+        '.css',
+        '.less',
+        '.png',
+        '.jpg',
+        '.jpeg',
+        '.gif',
+        '.pdf',
+        '.doc'
+      ]
+
+      @options = options
+      @options[:whitelist] = [@options[:whitelist]] if @options[:whitelist].is_a? String
+      @options[:blacklist] = [@options[:blacklist]] if @options[:blacklist].is_a? String
       @app = app
     end
 
@@ -23,36 +43,53 @@ module Rack
     end
 
     def should_show_prerendered_page(env)
+      user_agent = env['HTTP_USER_AGENT']
+      return false if !user_agent
+
       request = Rack::Request.new(env)
-      user_agent = env['HTTP_USER_AGENT'].downcase
 
-      should_show_based_on_agent_string = user_agent == 'googlebot' ||
-                        user_agent == 'yahoo' ||
-                        user_agent == 'bingbot' ||
-                        user_agent == 'baiduspider'
+      #if it is not a bot...dont prerender
+      return false if !@crawler_user_agents.include?(user_agent.downcase)
+
+      #if it is a bot and is requesting a resource...dont prerender
+      return false if @extensions_to_ignore.any? { |extension| request.path.include? extension }
+
+      #if it is a bot and not requesting a resource and is not whitelisted...dont prerender
+      return false if @options[:whitelist].is_a?(Array) && @options[:whitelist].all? { |whitelisted| !request.path.include? whitelisted }
 
-      return false if !should_show_based_on_agent_string #short circuit
+      #if it is a bot and not requesting a resource and is not blacklisted(url or referer)...dont prerender
+      if @options[:blacklist].is_a?(Array) && @options[:blacklist].any? { |blacklisted|
+          blacklistedUrl = false
+          blacklistedReferer = false
 
-      should_show_based_on_extension =  !request.path.include?('.js') &&
-                        !request.path.include?('.css') &&
-                        !request.path.include?('.less') &&
-                        !request.path.include?('.png') &&
-                        !request.path.include?('.jpg') &&
-                        !request.path.include?('.jpeg') &&
-                        !request.path.include?('.gif')
+          blacklistedUrl = request.path.include? blacklisted
+          blacklistedReferer = request.referer.include? blacklisted if request.referer
 
-      should_show_based_on_agent_string && should_show_based_on_extension
+          blacklistedUrl || blacklistedReferer
+        }
+        return false
+      end 
+
+      return true
     end
 
     def get_prerendered_page_response(env)
       begin
-        url = Rack::Request.new(env).url
-        prerender_url = ENV['PRERENDER_URL'] || 'http://prerender.herokuapp.com/'
-        forward_slash = prerender_url[-1, 1] == '/' ? '' : '/'
-        Net::HTTP.get_response(URI.parse("#{prerender_url}#{forward_slash}#{url}"))
+        Net::HTTP.get_response(URI.parse(build_api_url(env)))
       rescue
         nil
       end
     end
+
+    def build_api_url(env)
+      url = Rack::Request.new(env).url
+      prerender_url = get_prerender_service_url()
+      forward_slash = prerender_url[-1, 1] == '/' ? '' : '/'
+      "#{prerender_url}#{forward_slash}#{url}"
+    end
+
+    def get_prerender_service_url
+      ENV['PRERENDER_SERVICE_URL'] || 'http://prerender.herokuapp.com/'
+    end
   end
 end

data/prerender_rails.gemspec

@@ -1,13 +1,10 @@
 # coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'rack/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "prerender_rails"
-  spec.version       = PrerenderRails::VERSION
+  spec.version       = "0.0.3"
   spec.authors       = ["Todd Hooper"]
-  spec.email         = ["thoop3@gmail.com"]
+  spec.email         = ["todd@collectiveip.com"]
   spec.description   = %q{Rails middleware to prerender your javascript heavy pages on the fly by a phantomjs service}
   spec.summary       = %q{Prerender your backbone/angular/javascript rendered application on the fly when search engines crawl}
   spec.homepage      = "https://github.com/collectiveip/prerender_rails"
@@ -18,6 +15,9 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_dependency 'rack', '>= 0'
+
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "webmock"
 end

data/test/lib/prerender_rails.rb

@@ -0,0 +1,97 @@
+require_relative '../test_helper'
+ 
+describe Rack::Prerender do
+
+  bot = 'googlebot'
+  user = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.76 Safari/537.36'
+
+  before :each do
+    @app = lambda do |params|
+      [200, {}, ""]
+    end
+
+    @prerender = Rack::Prerender.new(@app)
+  end
+
+  it "should return a prerendered reponse for a crawler" do
+    request = Rack::MockRequest.env_for "/", "HTTP_USER_AGENT" => bot
+    stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
+    response = Rack::Prerender.new(@app).call(request)
+
+    assert_equal response[2].body, ["<html></html>"]
+  end
+
+  it "should continue to app routes if user is not a bot by checking agent string" do
+    request = Rack::MockRequest.env_for "/", "HTTP_USER_AGENT" => user
+    response = Rack::Prerender.new(@app).call(request)
+
+    assert_equal response[2], ""
+  end
+
+  it "should continue to app routes if user is a bot, but the bot is requesting a resource file" do
+    request = Rack::MockRequest.env_for "/main.js?anyQueryParam=true", "HTTP_USER_AGENT" => bot
+    response = Rack::Prerender.new(@app).call(request)
+
+    assert_equal response[2], ""
+  end
+
+  it "should continue to app routes if the url is not part of the whitelist" do
+    request = Rack::MockRequest.env_for "/profile/blah", "HTTP_USER_AGENT" => bot
+    response = Rack::Prerender.new(@app, whitelist: ['/search', '/help']).call(request)
+
+    assert_equal response[2], ""
+  end
+
+  it "should return a prerendered response if the url is part of the whitelist" do
+    request = Rack::MockRequest.env_for "/search/things?query=blah", "HTTP_USER_AGENT" => bot
+    stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
+    response = Rack::Prerender.new(@app, whitelist: ['/search', '/help']).call(request)
+
+    assert_equal response[2].body, ["<html></html>"]
+  end
+
+  it "should continue to app routes if the url is part of the blacklist" do
+    request = Rack::MockRequest.env_for "/search/things?query=blah", "HTTP_USER_AGENT" => bot
+    response = Rack::Prerender.new(@app, blacklist: ['/search', '/help']).call(request)
+
+    assert_equal response[2], ""
+  end
+
+  it "should return a prerendered response if the url is not part of the blacklist" do
+    request = Rack::MockRequest.env_for "/profile/blah", "HTTP_USER_AGENT" => bot
+    stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
+    response = Rack::Prerender.new(@app, blacklist: ['/search', '/help']).call(request)
+
+    assert_equal response[2].body, ["<html></html>"]
+  end
+
+  it "should continue to app routes if the referer is part of the blacklist" do
+    request = Rack::MockRequest.env_for "/api/results", "HTTP_USER_AGENT" => bot, "HTTP_REFERER" => '/search'
+    response = Rack::Prerender.new(@app, blacklist: ['/search', '/help']).call(request)
+
+    assert_equal response[2], ""
+  end
+
+  it "should return a prerendered response if the referer is not part of the blacklist" do
+    request = Rack::MockRequest.env_for "/api/results", "HTTP_USER_AGENT" => bot, "HTTP_REFERER" => '/search'
+    stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
+    response = Rack::Prerender.new(@app, blacklist: ['/profile', '/help']).call(request)
+
+    assert_equal response[2].body, ["<html></html>"]
+  end
+ 
+  describe '#buildApiUrl' do
+    it "should build the correct api url with the default url" do
+      request = Rack::MockRequest.env_for "https://google.com/search?q=javascript"
+      assert_equal @prerender.build_api_url(request), 'http://prerender.herokuapp.com/https://google.com/search?q=javascript'
+    end
+
+    it "should build the correct api url with an environment variable url" do
+      ENV['PRERENDER_SERVICE_URL'] = 'http://prerenderurl.com'
+      request = Rack::MockRequest.env_for "https://google.com/search?q=javascript"
+      assert_equal @prerender.build_api_url(request), 'http://prerenderurl.com/https://google.com/search?q=javascript'
+      ENV['PRERENDER_SERVICE_URL'] = nil
+    end
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require 'minitest/autorun'
+require 'webmock/minitest'
+require 'rack'
+require File.expand_path('../../lib/prerender_rails.rb', __FILE__)

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: prerender_rails
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Todd Hooper
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-27 00:00:00.000000000 Z
+date: 2013-09-29 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -38,22 +52,38 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Rails middleware to prerender your javascript heavy pages on the fly
   by a phantomjs service
 email:
-- thoop3@gmail.com
+- todd@collectiveip.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - lib/prerender_rails.rb
-- lib/rack/version.rb
 - prerender_rails.gemspec
+- test/lib/prerender_rails.rb
+- test/test_helper.rb
 homepage: https://github.com/collectiveip/prerender_rails
 licenses:
 - MIT
@@ -79,4 +109,6 @@ signing_key:
 specification_version: 4
 summary: Prerender your backbone/angular/javascript rendered application on the fly
   when search engines crawl
-test_files: []
+test_files:
+- test/lib/prerender_rails.rb
+- test/test_helper.rb

data/lib/rack/version.rb

@@ -1,3 +0,0 @@
-module PrerenderRails
-  VERSION = "0.0.2"
-end