prerender_rails 1.6.0 → 1.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/.travis.yml +3 -1
- data/README.md +11 -20
- data/lib/prerender_rails.rb +30 -2
- data/prerender_rails.gemspec +3 -3
- data/test/lib/prerender_rails.rb +48 -12
- metadata +21 -22
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 5b30d3022d29d47b8ab191bb223c669544cf0c91b2ae47d013e36e553d90b4b1
|
4
|
+
data.tar.gz: ad84a1c0b1e4a411575b53a17e9f6aa08445e264cf1c8216738ef79975ec1dcb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a41fde4e2d9372eb8a023e8fd9a56cd19944d2558da90d39caff0044aae21e0ee02ea7b3c1adaa26b0d757670e3c7f334b1b64cafb06dbe189411082646bfc21
|
7
|
+
data.tar.gz: 5b952c533002d2792f9f92c4fe21540349568fb82bcc2d23212466613321a706255b4aaae2e09b347784f4d5a1ee1fb582a83c9a96712cec14d935a99a1a371a
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -1,14 +1,11 @@
|
|
1
|
-
Prerender Rails [![
|
1
|
+
Prerender Rails [![Build Status](https://travis-ci.org/prerender/prerender_rails.png)](https://travis-ci.org/prerender/prerender_rails) [![Gem Version](https://badge.fury.io/rb/prerender_rails.png)](http://badge.fury.io/rb/prerender_rails)
|
2
2
|
===========================
|
3
3
|
|
4
|
-
Google, Facebook, Twitter,
|
4
|
+
Google, Facebook, Twitter, and Bing are constantly trying to view your website... but Google is the only crawler that executes a meaningful amount of JavaScript and Google even admits that they can execute JavaScript weeks after actually crawling. Prerender allows you to serve the full HTML of your website back to Google and other crawlers so that they don't have to execute any JavaScript. [Google recommends using Prerender.io](https://developers.google.com/search/docs/guides/dynamic-rendering) to prevent indexation issues on sites with large amounts of JavaScript.
|
5
5
|
|
6
|
-
|
6
|
+
Prerender is perfect for Angular SEO, React SEO, Vue SEO, and any other JavaScript framework.
|
7
7
|
|
8
|
-
|
9
|
-
- Just add <meta name="fragment" content="!"> to the <head> of all of your pages
|
10
|
-
- If you use hash urls (#), change them to the hash-bang (#!)
|
11
|
-
- That's it! Perfect SEO on javascript pages.
|
8
|
+
This middleware intercepts requests to your Node.js website from crawlers, and then makes a call to the (external) Prerender Service to get the static HTML instead of the JavaScript for that page. That HTML is then returned to the crawler.
|
12
9
|
|
13
10
|
`Note` Make sure you have more than one webserver thread/process running because the prerender service will make a request to your server to render the HTML.
|
14
11
|
|
@@ -22,7 +19,7 @@ And in `config/environment/production.rb`, add this line:
|
|
22
19
|
config.middleware.use Rack::Prerender
|
23
20
|
```
|
24
21
|
|
25
|
-
or if you have an account on [prerender.io](
|
22
|
+
or if you have an account on [prerender.io](https://prerender.io/) and want to use your token:
|
26
23
|
|
27
24
|
```ruby
|
28
25
|
config.middleware.use Rack::Prerender, prerender_token: 'YOUR_TOKEN'
|
@@ -32,21 +29,15 @@ or if you have an account on [prerender.io](http://prerender.io) and want to use
|
|
32
29
|
|
33
30
|
## Testing
|
34
31
|
|
35
|
-
When testing make sure you're not using a single threaded application server like default WEBrick one, use Puma or Unicorn.
|
32
|
+
When testing make sure you're not using a single threaded application server like default WEBrick one, use Puma or Unicorn to prevent a deadlock when the Prerender Service needs to render your HTML on the fly.
|
36
33
|
|
37
|
-
|
34
|
+
The best way to test the prerendered page is to [set the User Agent of your browser to Googlebot's user agent](https://developers.google.com/web/tools/chrome-devtools/device-mode/override-user-agent) and visit your URL directly. If you View Source on that URL, you should see the static HTML version of the page with the `<script>` tags removed from the page. If you still see `<script>` tags then that means the middleware isn't set up properly yet.
|
38
35
|
|
39
|
-
|
40
|
-
Then go to `http://localhost:5000/?_escaped_fragment_=/profiles/1234`
|
41
|
-
|
42
|
-
If your URLs use push-state:
|
43
|
-
|
44
|
-
If you want to see `http://localhost:5000/profiles/1234`
|
45
|
-
Then go to `http://localhost:5000/profiles/1234?_escaped_fragment_=`
|
36
|
+
`Note` If you're testing locally, you'll need to run the [prerender server](https://github.com/prerender/prerender) locally so that it has access to your server.
|
46
37
|
|
47
38
|
## How it works
|
48
39
|
1. The middleware checks to make sure we should show a prerendered page
|
49
|
-
1. The middleware checks if the request is from a crawler
|
40
|
+
1. The middleware checks if the request is from a crawler by checking the user agent string against a default list of crawler user agents
|
50
41
|
2. The middleware checks to make sure we aren't requesting a resource (js, css, etc...)
|
51
42
|
3. (optional) The middleware checks to make sure the url is in the whitelist
|
52
43
|
4. (optional) The middleware checks to make sure the url isn't in the blacklist
|
@@ -137,7 +128,7 @@ config.middleware.use Rack::Prerender,
|
|
137
128
|
|
138
129
|
## Using your own prerender service
|
139
130
|
|
140
|
-
We host a Prerender server at [prerender.io](
|
131
|
+
We host a Prerender server at [prerender.io](https://prerender.io/) so that you can work on more important things, but if you've deployed the prerender service on your own... set the `PRERENDER_SERVICE_URL` environment variable so that this middleware points there instead. Otherwise, it will default to the service already deployed by [prerender.io](https://prerender.io/).
|
141
132
|
|
142
133
|
$ export PRERENDER_SERVICE_URL=<new url>
|
143
134
|
|
@@ -148,7 +139,7 @@ Or on heroku:
|
|
148
139
|
As an alternative, you can pass `prerender_service_url` in the options object during initialization of the middleware
|
149
140
|
|
150
141
|
``` ruby
|
151
|
-
config.middleware.use Rack::Prerender, prerender_service_url: '
|
142
|
+
config.middleware.use Rack::Prerender, prerender_service_url: '<new url>'
|
152
143
|
```
|
153
144
|
|
154
145
|
## License
|
data/lib/prerender_rails.rb
CHANGED
@@ -4,6 +4,19 @@ module Rack
|
|
4
4
|
require 'active_support'
|
5
5
|
|
6
6
|
def initialize(app, options={})
|
7
|
+
@connection_header_values = [
|
8
|
+
'close',
|
9
|
+
'keep-alive'
|
10
|
+
].freeze
|
11
|
+
@hop_by_hop_headers = [
|
12
|
+
'Connection',
|
13
|
+
'Keep-Alive',
|
14
|
+
'Public',
|
15
|
+
'Proxy-Authenticate',
|
16
|
+
'Transfer-Encoding',
|
17
|
+
'Upgrade'
|
18
|
+
].freeze
|
19
|
+
|
7
20
|
@crawler_user_agents = [
|
8
21
|
'googlebot',
|
9
22
|
'yahoo',
|
@@ -34,7 +47,9 @@ module Rack
|
|
34
47
|
'nuzzel',
|
35
48
|
'Discordbot',
|
36
49
|
'Google Page Speed',
|
37
|
-
'Qwantify'
|
50
|
+
'Qwantify',
|
51
|
+
'Chrome-Lighthouse',
|
52
|
+
'TelegramBot'
|
38
53
|
]
|
39
54
|
|
40
55
|
@extensions_to_ignore = [
|
@@ -174,6 +189,19 @@ module Rack
|
|
174
189
|
response['Content-Length'] = response.body.length
|
175
190
|
response.delete('Content-Encoding')
|
176
191
|
end
|
192
|
+
|
193
|
+
hop_by_hop_headers = @hop_by_hop_headers
|
194
|
+
connection = response['Connection']
|
195
|
+
if connection
|
196
|
+
connection_hop_by_hop_headers = connection.split(',').
|
197
|
+
map(&:strip).
|
198
|
+
map(&:downcase).
|
199
|
+
difference(@connection_header_values)
|
200
|
+
hop_by_hop_headers = connection_hop_by_hop_headers.
|
201
|
+
concat(hop_by_hop_headers)
|
202
|
+
end
|
203
|
+
hop_by_hop_headers.each { |h| response.delete(h) }
|
204
|
+
|
177
205
|
response
|
178
206
|
rescue
|
179
207
|
nil
|
@@ -212,7 +240,7 @@ module Rack
|
|
212
240
|
|
213
241
|
|
214
242
|
def build_rack_response_from_prerender(prerendered_response)
|
215
|
-
response = Rack::Response.new(prerendered_response.body, prerendered_response.code, prerendered_response
|
243
|
+
response = Rack::Response.new(prerendered_response.body, prerendered_response.code, prerendered_response)
|
216
244
|
|
217
245
|
@options[:build_rack_response_from_prerender].call(response, prerendered_response) if @options[:build_rack_response_from_prerender]
|
218
246
|
|
data/prerender_rails.gemspec
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |spec|
|
4
4
|
spec.name = "prerender_rails"
|
5
|
-
spec.version = "1.
|
5
|
+
spec.version = "1.8.0"
|
6
6
|
spec.authors = ["Todd Hooper"]
|
7
7
|
spec.email = ["todd@prerender.io"]
|
8
8
|
spec.description = %q{Rails middleware to prerender your javascript heavy pages on the fly by a phantomjs service}
|
@@ -15,10 +15,10 @@ Gem::Specification.new do |spec|
|
|
15
15
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
16
16
|
spec.require_paths = ["lib"]
|
17
17
|
|
18
|
-
spec.add_dependency 'rack', '
|
18
|
+
spec.add_dependency 'rack', '~> 2.2.2'
|
19
19
|
spec.add_dependency 'activesupport', '>= 0'
|
20
20
|
|
21
|
-
spec.add_development_dependency "bundler", "~> 1.
|
21
|
+
spec.add_development_dependency "bundler", "~> 2.1.2"
|
22
22
|
spec.add_development_dependency "rake"
|
23
23
|
spec.add_development_dependency "webmock"
|
24
24
|
end
|
data/test/lib/prerender_rails.rb
CHANGED
@@ -19,9 +19,9 @@ describe Rack::Prerender do
|
|
19
19
|
stub_request(:get, @prerender.build_api_url(request)).with(:headers => { 'User-Agent' => bot }).to_return(:body => "<html></html>", :status => 301, :headers => { 'Location' => 'http://google.com'})
|
20
20
|
response = Rack::Prerender.new(@app).call(request)
|
21
21
|
|
22
|
-
assert_equal response[2]
|
23
|
-
assert_equal response[
|
24
|
-
assert_equal( { 'location' => 'http://google.com'
|
22
|
+
assert_equal response[2], ["<html></html>"]
|
23
|
+
assert_equal response[0], 301
|
24
|
+
assert_equal( { 'location' => 'http://google.com'}, response[1] )
|
25
25
|
end
|
26
26
|
|
27
27
|
|
@@ -30,7 +30,7 @@ describe Rack::Prerender do
|
|
30
30
|
stub_request(:get, @prerender.build_api_url(request)).with(:headers => { 'User-Agent' => user }).to_return(:body => "<html></html>")
|
31
31
|
response = Rack::Prerender.new(@app).call(request)
|
32
32
|
|
33
|
-
assert_equal ["<html></html>"], response[2]
|
33
|
+
assert_equal ["<html></html>"], response[2]
|
34
34
|
end
|
35
35
|
|
36
36
|
|
@@ -89,7 +89,7 @@ describe Rack::Prerender do
|
|
89
89
|
stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
|
90
90
|
response = @prerender.call(request)
|
91
91
|
|
92
|
-
assert_equal ["<html></html>"], response[2]
|
92
|
+
assert_equal ["<html></html>"], response[2]
|
93
93
|
end
|
94
94
|
|
95
95
|
|
@@ -98,7 +98,7 @@ describe Rack::Prerender do
|
|
98
98
|
stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
|
99
99
|
response = Rack::Prerender.new(@app, whitelist: ['^/search.*page', '/help']).call(request)
|
100
100
|
|
101
|
-
assert_equal ["<html></html>"], response[2]
|
101
|
+
assert_equal ["<html></html>"], response[2]
|
102
102
|
end
|
103
103
|
|
104
104
|
|
@@ -121,7 +121,7 @@ describe Rack::Prerender do
|
|
121
121
|
stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
|
122
122
|
response = Rack::Prerender.new(@app, blacklist: ['^/search', '/help']).call(request)
|
123
123
|
|
124
|
-
assert_equal ["<html></html>"], response[2]
|
124
|
+
assert_equal ["<html></html>"], response[2]
|
125
125
|
end
|
126
126
|
|
127
127
|
|
@@ -138,7 +138,7 @@ describe Rack::Prerender do
|
|
138
138
|
stub_request(:get, @prerender.build_api_url(request)).to_return(:body => "<html></html>")
|
139
139
|
response = Rack::Prerender.new(@app, blacklist: ['^/search', '/help']).call(request)
|
140
140
|
|
141
|
-
assert_equal ["<html></html>"], response[2]
|
141
|
+
assert_equal ["<html></html>"], response[2]
|
142
142
|
end
|
143
143
|
|
144
144
|
|
@@ -146,7 +146,7 @@ describe Rack::Prerender do
|
|
146
146
|
request = Rack::MockRequest.env_for "/", "HTTP_USER_AGENT" => bot
|
147
147
|
response = Rack::Prerender.new(@app, before_render: Proc.new do |env| '<html>cached</html>' end).call(request)
|
148
148
|
|
149
|
-
assert_equal ["<html>cached</html>"], response[2]
|
149
|
+
assert_equal ["<html>cached</html>"], response[2]
|
150
150
|
end
|
151
151
|
|
152
152
|
|
@@ -154,11 +154,47 @@ describe Rack::Prerender do
|
|
154
154
|
request = Rack::MockRequest.env_for "/", "HTTP_USER_AGENT" => bot
|
155
155
|
response = Rack::Prerender.new(@app, before_render: Proc.new do |env| Rack::Response.new('<html>cached2</html>', 200, { 'test' => 'test2Header'}) end).call(request)
|
156
156
|
|
157
|
-
assert_equal ["<html>cached2</html>"], response[2]
|
158
|
-
assert_equal response[
|
159
|
-
assert_equal( { 'test' => 'test2Header'
|
157
|
+
assert_equal ["<html>cached2</html>"], response[2]
|
158
|
+
assert_equal response[0], 200
|
159
|
+
assert_equal( { 'test' => 'test2Header'}, response[1] )
|
160
160
|
end
|
161
161
|
|
162
|
+
it "should return a prerendered response stripped of hop-by-hop headers" do
|
163
|
+
request = Rack::MockRequest.env_for "/", "HTTP_USER_AGENT" => bot
|
164
|
+
stub_request(:get, @prerender.build_api_url(request)).
|
165
|
+
with(:headers => { 'User-Agent' => bot }).
|
166
|
+
to_return(:body => "<html></html>", :status => 401, :headers => {
|
167
|
+
'Content-Type' => 'text/html',
|
168
|
+
'Transfer-Encoding' => 'Chunked',
|
169
|
+
'Connection' => 'Keep-Alive',
|
170
|
+
'Keep-Alive' => 'timeout=5, max=100',
|
171
|
+
'Public' => 'GET HEAD',
|
172
|
+
'Proxy-Authenticate' => 'Basic',
|
173
|
+
'X-Drop-Test' => 'ShouldAlwaysHappen',
|
174
|
+
'Upgrade' => 'dummy'
|
175
|
+
})
|
176
|
+
response = Rack::Prerender.new(@app).call(request)
|
177
|
+
|
178
|
+
assert_equal response[2], ["<html></html>"]
|
179
|
+
assert_equal response[0], 401
|
180
|
+
assert_equal( { 'content-type' => 'text/html', 'x-drop-test' => 'ShouldAlwaysHappen'}, response[1] )
|
181
|
+
end
|
182
|
+
|
183
|
+
it "should return a prerendered response stripped of custom-defined hop-by-hop headers" do
|
184
|
+
request = Rack::MockRequest.env_for "/", "HTTP_USER_AGENT" => bot
|
185
|
+
stub_request(:get, @prerender.build_api_url(request)).
|
186
|
+
with(:headers => { 'User-Agent' => bot }).
|
187
|
+
to_return(:body => "<html></html>", :status => 200, :headers => {
|
188
|
+
'Content-Type' => 'text/html',
|
189
|
+
'Connection' => 'Close, X-Drop-Test',
|
190
|
+
'X-Drop-Test' => 'ShouldNeverHappen'
|
191
|
+
})
|
192
|
+
response = Rack::Prerender.new(@app).call(request)
|
193
|
+
|
194
|
+
assert_equal response[2], ["<html></html>"]
|
195
|
+
assert_equal response[0], 200
|
196
|
+
assert_equal( { 'content-type' => 'text/html' }, response[1] )
|
197
|
+
end
|
162
198
|
|
163
199
|
describe '#buildApiUrl' do
|
164
200
|
it "should build the correct api url with the default url" do
|
metadata
CHANGED
@@ -1,83 +1,83 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: prerender_rails
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Todd Hooper
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-11-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rack
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 2.2.2
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: 2.2.2
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: activesupport
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: bundler
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - ~>
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 2.1.2
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - ~>
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 2.1.2
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rake
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: webmock
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
description: Rails middleware to prerender your javascript heavy pages on the fly
|
@@ -88,8 +88,8 @@ executables: []
|
|
88
88
|
extensions: []
|
89
89
|
extra_rdoc_files: []
|
90
90
|
files:
|
91
|
-
- .gitignore
|
92
|
-
- .travis.yml
|
91
|
+
- ".gitignore"
|
92
|
+
- ".travis.yml"
|
93
93
|
- Gemfile
|
94
94
|
- LICENSE.txt
|
95
95
|
- README.md
|
@@ -108,17 +108,16 @@ require_paths:
|
|
108
108
|
- lib
|
109
109
|
required_ruby_version: !ruby/object:Gem::Requirement
|
110
110
|
requirements:
|
111
|
-
- -
|
111
|
+
- - ">="
|
112
112
|
- !ruby/object:Gem::Version
|
113
113
|
version: '0'
|
114
114
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
115
115
|
requirements:
|
116
|
-
- -
|
116
|
+
- - ">="
|
117
117
|
- !ruby/object:Gem::Version
|
118
118
|
version: '0'
|
119
119
|
requirements: []
|
120
|
-
|
121
|
-
rubygems_version: 2.0.14.1
|
120
|
+
rubygems_version: 3.0.3.1
|
122
121
|
signing_key:
|
123
122
|
specification_version: 4
|
124
123
|
summary: Prerender your backbone/angular/javascript rendered application on the fly
|