prerender_rails 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd646cf6503266ed2db50c786e116d740e63c4c9
-  data.tar.gz: 07cea4440350844c52ae3a29a483a00baefc1dba
+  metadata.gz: 1773ce49f7fc96e82cc3a4186c4c9eb51f3c2479
+  data.tar.gz: 12ab0d5981038d66d85f5198e35f2592a46a2b74
 SHA512:
-  metadata.gz: a37f41bb0b7535a9b0386191a231697be90e468614875dd4cf8a3b75781407ca8784b33051f08e539eb0a18f5d345f0bde8c611896abcd5faf08e6862efe3909
-  data.tar.gz: 447741f462ae27b1a4f11fc8a4f6c72ffbc92153e69a682c0eb3b24e74f6ba007166bc574fdc1388c49f61efa1dea7b4d407c1316ba0976bcc4e1e1eab52012c
+  metadata.gz: cdabf0df7454ac647756a966a2f7d3eecafa04f26fa2d293e278722442d1aaa5fcefcb07d0859a5789b178b730b54d4053ecfc9c55a3777e8b0eddd3527ee7d8
+  data.tar.gz: 52ffb587b5a76c393c44859477a693e675718d52f4ad180d9725f54b53ea28936cb2d9c5d6d0e75be28421c7041cc9ce4702e6a4fbf50c7e597657f3581ae4ea

data/README.md

@@ -5,9 +5,11 @@ Are you using backbone, angular, emberjs, etc, but you're unsure about the SEO i
 
 Use this gem to install rails middleware that prerenders a javascript-rendered page using an external service and returns the HTML to the search engine crawler for SEO.
 
-`Note:` If you are using a `#` in your urls, make sure to change it to `#!`. [View Google's ajax crawling protocol](https://developers.google.com/webmasters/ajax-crawling/docs/getting-started)
+`Note:`
 
-`Note:` Make sure you have more than one webserver thread/process running because the prerender service will make a request to your server to render the HTML.
+* If you are using a `#` in your urls, make sure to change it to `#!`. [View Google's ajax crawling protocol](https://developers.google.com/webmasters/ajax-crawling/docs/getting-started)
+* Make sure you have more than one webserver thread/process running because the prerender service will make a request to your server to render the HTML.
+* If you're testing on localhost, you need to run the prerender server locally so that it can access your server.
 
 Add this line to your application's Gemfile:
 
@@ -28,6 +30,29 @@ And in `config/environment/production.rb`, add this line:
 2. Make a `GET` request to the [prerender service](https://github.com/collectiveip/prerender)(phantomjs server) for the page's prerendered HTML
 3. Return that HTML to the crawler
 
+
+## Caching
+
+This rails middleware is ready to be used with [redis](http://redis.io/) or [memcached](http://memcached.org/) to return prerendered pages in milliseconds.
+
+When setting up the middleware in `config/environment/production.rb`, you can add a `before_render` method and `after_render` method for caching.
+
+Here's an example testing a local redis cache:
+
+_Put this in `config/environment/development.rb`, and add `gem 'redis'` to your Gemfile._
+
+```ruby
+require 'redis'
+@redis = Redis.new
+config.middleware.use Rack::Prerender,
+  before_render: (Proc.new do |env|
+    @redis.get(Rack::Request.new(env).url)
+  end),
+  after_render: (Proc.new do |env, response|
+    @redis.set(Rack::Request.new(env).url, response.body)
+  end)
+```
+
 ## Customization
 
 ### Whitelist
@@ -50,7 +75,38 @@ config.middleware.use Rack::Prerender, blacklist: '^/search'
 config.middleware.use Rack::Prerender, blacklist: ['/search', '/users/.*/profile']
 ```
 
-### Using your own prerender service
+### before_render
+
+This method is intended to be used for caching, but could be used to save analytics or anything else you need to do for each crawler request. If you return a string from before_render, the middleware will server that to the crawler instead of making a request to the prerender service.
+```ruby
+config.middleware.use Rack::Prerender,
+	before_render: (Proc.new do |env|
+		# do whatever you need to do.
+	end)
+```
+
+### after_render
+
+This method is intended to be used for caching, but could be used to save analytics or anything else you need to do for each crawler request. This method is a noop and is called after the prerender service returns HTML.
+```ruby
+config.middleware.use Rack::Prerender,
+	after_render: (Proc.new do |env, response|
+		# do whatever you need to do.
+	end)
+```
+
+### build_rack_response_from_prerender
+
+This method is intended to be used to modify the response before it is sent to the crawler. Use this method to add/remove response headers, or do anything else before the request is sent.
+```ruby
+config.middleware.use Rack::Prerender,
+	build_rack_response_from_prerender: (Proc.new do |response, prerender_response|
+		# response is already populated with the prerender status code, html, and headers
+		# prerender_response is the response that came back from the prerender service
+	end)
+```
+
+## Using your own prerender service
 
 If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://prerender.herokuapp.com`
 

data/lib/prerender_rails.rb

@@ -62,13 +62,21 @@ module Rack
       @app = app
     end
 
-    def call(env)
 
+    def call(env)
       if should_show_prerendered_page(env)
+
+        cached_response = before_render(env)
+
+        if cached_response && cached_response.is_a?(Rack::Response)
+          return cached_response.finish
+        end
+
         prerendered_response = get_prerendered_page_response(env)
 
         if prerendered_response
-          response = Rack::Response.new(prerendered_response.body, prerendered_response.code, prerendered_response.header)
+          response = build_rack_response_from_prerender(prerendered_response)
+          after_render(env, prerendered_response)
           return response.finish
         end
       end
@@ -76,6 +84,7 @@ module Rack
       @app.call(env)  
     end
 
+
     def should_show_prerendered_page(env)
       user_agent = env['HTTP_USER_AGENT']
       return false if !user_agent
@@ -111,6 +120,7 @@ module Rack
       return true
     end
 
+
     def get_prerendered_page_response(env)
       begin
         url = URI.parse(build_api_url(env))
@@ -123,6 +133,7 @@ module Rack
       end
     end
 
+
     def build_api_url(env)
       new_env = env
       if env["CF-VISITOR"]
@@ -137,8 +148,37 @@ module Rack
       "#{prerender_url}#{forward_slash}#{url}"
     end
 
+
     def get_prerender_service_url
       @options[:prerender_service_url] || ENV['PRERENDER_SERVICE_URL'] || 'http://prerender.herokuapp.com/'
     end
+
+
+    def build_rack_response_from_prerender(prerendered_response)
+      response = Rack::Response.new(prerendered_response.body, prerendered_response.code, prerendered_response.header)
+
+      @options[:build_rack_response_from_prerender].call(response, prerendered_response) if @options[:build_rack_response_from_prerender]
+
+      response
+    end
+
+
+    def before_render(env)
+      return nil unless @options[:before_render]
+
+      cached_render = @options[:before_render].call(env)
+      
+      if cached_render && cached_render.is_a?(String)
+        Rack::Response.new(cached_render, 200, [])
+      else
+        nil
+      end
+    end
+
+
+    def after_render(env, response)
+      return true unless @options[:after_render]
+      @options[:after_render].call(env, response)
+    end
   end
 end

data/prerender_rails.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "prerender_rails"
-  spec.version       = "0.1.9"
+  spec.version       = "0.1.10"
   spec.authors       = ["Todd Hooper"]
   spec.email         = ["todd@collectiveip.com"]
   spec.description   = %q{Rails middleware to prerender your javascript heavy pages on the fly by a phantomjs service}

data/test/lib/prerender_rails.rb

@@ -13,6 +13,7 @@ describe Rack::Prerender do
     @prerender = Rack::Prerender.new(@app)
   end
 
+
   it "should return a prerendered response for a crawler with the returned status code and headers" do
     request = Rack::MockRequest.env_for "/", "HTTP_USER_AGENT" => bot
     stub_request(:get, @prerender.build_api_url(request)).with(:headers => { 'User-Agent' => bot }).to_return(:body => "<html></html>", :status => 301, :headers => { 'Location' => 'http://google.com'})
@@ -23,6 +24,7 @@ describe Rack::Prerender do
     assert_equal( { 'location' => 'http://google.com', 'Content-Length' => '13'}, response[2].headers )
   end
 
+
   it "should return a prerendered reponse if user is a bot by checking for _escaped_fragment_" do
     request = Rack::MockRequest.env_for "/path?_escaped_fragment_=", "HTTP_USER_AGENT" => user
     stub_request(:get, @prerender.build_api_url(request)).with(:headers => { 'User-Agent' => user }).to_return(:body => "<html></html>")
@@ -31,6 +33,7 @@ describe Rack::Prerender do
     assert_equal ["<html></html>"], response[2].body
   end
 
+
   it "should continue to app routes if the url is a bad url with _escaped_fragment_" do
     request = Rack::MockRequest.env_for "/path?query=string?_escaped_fragment_=", "HTTP_USER_AGENT" => user
     response = Rack::Prerender.new(@app).call(request)
@@ -38,6 +41,7 @@ describe Rack::Prerender do
     assert_equal "", response[2]
   end
 
+
   it "should continue to app routes if the request is not a GET" do
     request = Rack::MockRequest.env_for "/path?_escaped_fragment_=", { "HTTP_USER_AGENT" => user, "REQUEST_METHOD" => "POST" }
     response = Rack::Prerender.new(@app).call(request)
@@ -45,6 +49,7 @@ describe Rack::Prerender do
     assert_equal "", response[2]
   end
 
+
   it "should continue to app routes if user is not a bot by checking agent string" do
     request = Rack::MockRequest.env_for "/", "HTTP_USER_AGENT" => user
     response = Rack::Prerender.new(@app).call(request)
@@ -52,6 +57,7 @@ describe Rack::Prerender do
     assert_equal "", response[2]
   end
 
+
   it "should continue to app routes if user is a bot, but the bot is requesting a resource file" do
     request = Rack::MockRequest.env_for "/main.js?anyQueryParam=true", "HTTP_USER_AGENT" => bot
     response = Rack::Prerender.new(@app).call(request)
@@ -59,6 +65,7 @@ describe Rack::Prerender do
     assert_equal "", response[2]
   end
 
+
   it "should continue to app routes if the url is not part of the regex specific whitelist" do
     request = Rack::MockRequest.env_for "/saved/search/blah", "HTTP_USER_AGENT" => bot
     response = Rack::Prerender.new(@app, whitelist: ['^/search', '/help']).call(request)
@@ -66,6 +73,7 @@ describe Rack::Prerender do
     assert_equal "", response[2]
   end
 
+
   it "should return a prerendered response if the url is part of the regex specific whitelist" do
     request = Rack::MockRequest.env_for "/search/things/123/page", "HTTP_USER_AGENT" => bot
     stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
@@ -74,6 +82,7 @@ describe Rack::Prerender do
     assert_equal ["<html></html>"], response[2].body
   end
 
+
   it "should continue to app routes if the url is part of the regex specific blacklist" do
     request = Rack::MockRequest.env_for "/search/things/123/page", "HTTP_USER_AGENT" => bot
     response = Rack::Prerender.new(@app, blacklist: ['^/search', '/help']).call(request)
@@ -81,6 +90,7 @@ describe Rack::Prerender do
     assert_equal "", response[2]
   end
 
+
   it "should return a prerendered response if the url is not part of the regex specific blacklist" do
     request = Rack::MockRequest.env_for "/profile/search/blah", "HTTP_USER_AGENT" => bot
     stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
@@ -89,6 +99,7 @@ describe Rack::Prerender do
     assert_equal ["<html></html>"], response[2].body
   end
 
+
   it "should continue to app routes if the referer is part of the regex specific blacklist" do
     request = Rack::MockRequest.env_for "/api/results", "HTTP_USER_AGENT" => bot, "HTTP_REFERER" => '/search'
     response = Rack::Prerender.new(@app, blacklist: ['^/search', '/help']).call(request)
@@ -96,6 +107,7 @@ describe Rack::Prerender do
     assert_equal "", response[2]
   end
 
+
   it "should return a prerendered response if the referer is not part of the regex specific blacklist" do
     request = Rack::MockRequest.env_for "/api/results", "HTTP_USER_AGENT" => bot, "HTTP_REFERER" => '/profile/search'
     stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
@@ -103,6 +115,16 @@ describe Rack::Prerender do
 
     assert_equal ["<html></html>"], response[2].body
   end
+
+
+  it "should return a prerendered response returned from before_render" do
+    request = Rack::MockRequest.env_for "/", "HTTP_USER_AGENT" => bot
+    stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
+    response = Rack::Prerender.new(@app, before_render: Proc.new do |env| return '<html>cached</html>' end).call(request)
+
+    assert_equal ["<html>cached</html>"], response[2].body
+  end
+
  
   describe '#buildApiUrl' do
     it "should build the correct api url with the default url" do
@@ -111,6 +133,7 @@ describe Rack::Prerender do
       assert_equal 'http://prerender.herokuapp.com/https://google.com/search?q=javascript', @prerender.build_api_url(request)
     end
 
+
     it "should build the correct api url with an environment variable url" do
       ENV['PRERENDER_SERVICE_URL'] = 'http://prerenderurl.com'
       request = Rack::MockRequest.env_for "https://google.com/search?q=javascript"
@@ -118,12 +141,14 @@ describe Rack::Prerender do
       ENV['PRERENDER_SERVICE_URL'] = nil
     end
 
+
     it "should build the correct api url with an initialization variable url" do
       @prerender = Rack::Prerender.new(@app, prerender_service_url: 'http://prerenderurl.com')
       request = Rack::MockRequest.env_for "https://google.com/search?q=javascript"
       assert_equal 'http://prerenderurl.com/https://google.com/search?q=javascript', @prerender.build_api_url(request)
     end
 
+
     # Check CF-Visitor header in order to Work behind CloudFlare with Flexible SSL (https://support.cloudflare.com/hc/en-us/articles/200170536)
     it "should build the correct api url for the Cloudflare Flexible SSL support" do
       request = Rack::MockRequest.env_for "http://google.com/search?q=javascript", { 'CF-VISITOR' => '"scheme":"https"'}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prerender_rails
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Todd Hooper
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-16 00:00:00.000000000 Z
+date: 2013-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack