rack-prerender 1.6.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/rack/prerender/constraint.rb +122 -0
- data/lib/rack/prerender/fetcher.rb +98 -0
- data/lib/rack/prerender/version.rb +7 -0
- data/lib/rack/prerender.rb +20 -0
- metadata +121 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 0455bf3050d53db38d75d2162014caf79f05526e89b754742a612c0dd29bde55
|
4
|
+
data.tar.gz: 34085238abec2b269ea584e02678d302b2fa69fd09ea4c2570490f481b804eb1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b30c8d6f9612ad4e7b98ed59f2ee51511a18e2d7bc92ff2a37b2a7fa40dca8d205af150fa30fe915de82b63fd8d66246dceb689d28b4814cc3a705200050c7ea
|
7
|
+
data.tar.gz: dae2fb0e2a877abebf84200b09318a15479459776a6496553ffd499e20a3e7d0d97ce0d23b04adbd5446d8273afbff34bf2d1e255519a5806876dc5f0e5b9952
|
@@ -0,0 +1,122 @@
|
|
1
|
+
module Rack
|
2
|
+
class Prerender
|
3
|
+
class Constraint
|
4
|
+
attr_reader :blacklist, :whitelist, :crawler_user_agents, :extensions_to_ignore
|
5
|
+
|
6
|
+
def initialize(opts = {})
|
7
|
+
@blacklist = cast_to_regexp(opts[:blacklist], escape: false)
|
8
|
+
@whitelist = cast_to_regexp(opts[:whitelist], escape: false)
|
9
|
+
@crawler_user_agents = cast_to_regexp(opts[:crawler_user_agents] || CRAWLER_USER_AGENTS)
|
10
|
+
@extensions_to_ignore = cast_to_regexp(opts[:extensions_to_ignore] || EXTENSIONS_TO_IGNORE)
|
11
|
+
end
|
12
|
+
|
13
|
+
def matches?(env)
|
14
|
+
return false if env['REQUEST_METHOD'] != 'GET' ||
|
15
|
+
env['HTTP_X_PRERENDER'] ||
|
16
|
+
(user_agent = env['HTTP_USER_AGENT']).nil?
|
17
|
+
|
18
|
+
query = env['QUERY_STRING'].to_s
|
19
|
+
return false unless crawler_user_agents.match?(user_agent.downcase) ||
|
20
|
+
env['HTTP_X_BUFFERBOT'] ||
|
21
|
+
query.include?('_escaped_fragment_')
|
22
|
+
|
23
|
+
path = env['SCRIPT_NAME'].to_s + env['PATH_INFO'].to_s
|
24
|
+
fullpath = query.empty? ? path : "#{path}?#{query}"
|
25
|
+
return false if extensions_to_ignore.match?(fullpath)
|
26
|
+
return false if whitelist && !whitelist.match?(fullpath)
|
27
|
+
return false if blacklist && (blacklist.match?(fullpath) ||
|
28
|
+
blacklist.match?(env['HTTP_REFERER'].to_s.downcase))
|
29
|
+
|
30
|
+
true
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def cast_to_regexp(list_arg, escape: true)
|
36
|
+
case list_arg
|
37
|
+
when Regexp, nil
|
38
|
+
list_arg
|
39
|
+
when Array
|
40
|
+
escape ? Regexp.union(list_arg) : Regexp.new(list_arg.join('|'))
|
41
|
+
else
|
42
|
+
Regexp.new(escape ? Regexp.escape(list_arg) : list_arg)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
CRAWLER_USER_AGENTS = [
|
47
|
+
'applebot',
|
48
|
+
'baiduspider',
|
49
|
+
'bingbot',
|
50
|
+
'bitlybot',
|
51
|
+
'bufferbot',
|
52
|
+
'chrome-lighthouse',
|
53
|
+
'developers.google.com/+/web/snippet',
|
54
|
+
'discordbot',
|
55
|
+
'embedly',
|
56
|
+
'facebookexternalhit',
|
57
|
+
'flipboard',
|
58
|
+
'google page speed',
|
59
|
+
'googlebot',
|
60
|
+
'linkedinbot',
|
61
|
+
'nuzzel',
|
62
|
+
'outbrain',
|
63
|
+
'pinterest/0.',
|
64
|
+
'quora link preview',
|
65
|
+
'qwantify',
|
66
|
+
'redditbot',
|
67
|
+
'rogerbot',
|
68
|
+
'showyoubot',
|
69
|
+
'skypeuripreview',
|
70
|
+
'slackbot',
|
71
|
+
'tumblr',
|
72
|
+
'twitterbot',
|
73
|
+
'vkshare',
|
74
|
+
'w3c_validator',
|
75
|
+
'whatsapp',
|
76
|
+
'www.google.com/webmasters/tools/richsnippets',
|
77
|
+
'yahoo',
|
78
|
+
]
|
79
|
+
|
80
|
+
EXTENSIONS_TO_IGNORE = %w[
|
81
|
+
.ai
|
82
|
+
.avi
|
83
|
+
.css
|
84
|
+
.dat
|
85
|
+
.dmg
|
86
|
+
.doc
|
87
|
+
.doc
|
88
|
+
.exe
|
89
|
+
.flv
|
90
|
+
.gif
|
91
|
+
.ico
|
92
|
+
.iso
|
93
|
+
.jpeg
|
94
|
+
.jpg
|
95
|
+
.js
|
96
|
+
.less
|
97
|
+
.m4a
|
98
|
+
.m4v
|
99
|
+
.mov
|
100
|
+
.mp3
|
101
|
+
.mp4
|
102
|
+
.mpeg
|
103
|
+
.mpg
|
104
|
+
.pdf
|
105
|
+
.png
|
106
|
+
.ppt
|
107
|
+
.psd
|
108
|
+
.rar
|
109
|
+
.rss
|
110
|
+
.swf
|
111
|
+
.tif
|
112
|
+
.torrent
|
113
|
+
.txt
|
114
|
+
.wav
|
115
|
+
.wmv
|
116
|
+
.xls
|
117
|
+
.xml
|
118
|
+
.zip
|
119
|
+
]
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
module Rack
|
2
|
+
class Prerender
|
3
|
+
class Fetcher
|
4
|
+
attr_reader :env, :options
|
5
|
+
|
6
|
+
def initialize(options = {})
|
7
|
+
@options = options
|
8
|
+
end
|
9
|
+
|
10
|
+
def call(env)
|
11
|
+
cached_response = before_render(env)
|
12
|
+
return cached_response.finish if cached_response
|
13
|
+
|
14
|
+
if prerendered_response = fetch(env)
|
15
|
+
response = build_rack_response_from_prerender(prerendered_response)
|
16
|
+
after_render(env, prerendered_response)
|
17
|
+
return response.finish
|
18
|
+
end
|
19
|
+
nil
|
20
|
+
end
|
21
|
+
|
22
|
+
def before_render(env)
|
23
|
+
return unless callback = options[:before_render]
|
24
|
+
|
25
|
+
cached_render = callback.call(env)
|
26
|
+
|
27
|
+
if cached_render && cached_render.is_a?(String)
|
28
|
+
Rack::Response.new(cached_render, 200, { 'Content-Type' => 'text/html; charset=utf-8' })
|
29
|
+
elsif cached_render && cached_render.is_a?(Rack::Response)
|
30
|
+
cached_render
|
31
|
+
else
|
32
|
+
nil
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def fetch(env)
|
37
|
+
url = URI.parse(api_url(env))
|
38
|
+
headers = {
|
39
|
+
'User-Agent' => env['HTTP_USER_AGENT'],
|
40
|
+
'Accept-Encoding' => 'gzip'
|
41
|
+
}
|
42
|
+
headers['X-Prerender-Token'] = ENV['PRERENDER_TOKEN'] if ENV['PRERENDER_TOKEN']
|
43
|
+
headers['X-Prerender-Token'] = options[:prerender_token] if options[:prerender_token]
|
44
|
+
req = Net::HTTP::Get.new(url.request_uri, headers)
|
45
|
+
req.basic_auth(ENV['PRERENDER_USERNAME'], ENV['PRERENDER_PASSWORD']) if options[:basic_auth]
|
46
|
+
http = Net::HTTP.new(url.host, url.port)
|
47
|
+
http.use_ssl = true if url.scheme == 'https'
|
48
|
+
response = http.request(req)
|
49
|
+
if response['Content-Encoding'] == 'gzip'
|
50
|
+
response.body = ActiveSupport::Gzip.decompress(response.body)
|
51
|
+
response['Content-Length'] = response.body.length
|
52
|
+
response.delete('Content-Encoding')
|
53
|
+
end
|
54
|
+
response
|
55
|
+
rescue
|
56
|
+
nil
|
57
|
+
end
|
58
|
+
|
59
|
+
def api_url(env)
|
60
|
+
new_env = env
|
61
|
+
if env['CF-VISITOR']
|
62
|
+
match = /"scheme":"(http|https)"/.match(env['CF-VISITOR'])
|
63
|
+
new_env['HTTPS'] = true and new_env['rack.url_scheme'] = "https" and new_env['SERVER_PORT'] = 443 if (match && match[1] == 'https')
|
64
|
+
new_env['HTTPS'] = false and new_env['rack.url_scheme'] = "http" and new_env['SERVER_PORT'] = 80 if (match && match[1] == 'http')
|
65
|
+
end
|
66
|
+
|
67
|
+
if env['X-FORWARDED-PROTO']
|
68
|
+
new_env['HTTPS'] = true and new_env['rack.url_scheme'] = "https" and new_env['SERVER_PORT'] = 443 if env["X-FORWARDED-PROTO"].split(',')[0] == 'https'
|
69
|
+
new_env['HTTPS'] = false and new_env['rack.url_scheme'] = "http" and new_env['SERVER_PORT'] = 80 if env["X-FORWARDED-PROTO"].split(',')[0] == 'http'
|
70
|
+
end
|
71
|
+
|
72
|
+
if options[:protocol]
|
73
|
+
new_env['HTTPS'] = true and new_env['rack.url_scheme'] = "https" and new_env['SERVER_PORT'] = 443 if options[:protocol] == 'https'
|
74
|
+
new_env['HTTPS'] = false and new_env['rack.url_scheme'] = "http" and new_env['SERVER_PORT'] = 80 if options[:protocol] == 'http'
|
75
|
+
end
|
76
|
+
|
77
|
+
url = Rack::Request.new(new_env).url
|
78
|
+
prerender_url = prerender_service_url()
|
79
|
+
forward_slash = prerender_url[-1, 1] == '/' ? '' : '/'
|
80
|
+
"#{prerender_url}#{forward_slash}#{url}"
|
81
|
+
end
|
82
|
+
|
83
|
+
def prerender_service_url
|
84
|
+
options[:prerender_service_url] || ENV['PRERENDER_SERVICE_URL'] || 'http://service.prerender.io/'
|
85
|
+
end
|
86
|
+
|
87
|
+
def build_rack_response_from_prerender(prerendered_response)
|
88
|
+
response = Rack::Response.new(prerendered_response.body, prerendered_response.code, prerendered_response)
|
89
|
+
options[:build_rack_response_from_prerender].call(response, prerendered_response) if options[:build_rack_response_from_prerender]
|
90
|
+
response
|
91
|
+
end
|
92
|
+
|
93
|
+
def after_render(env, response)
|
94
|
+
(callback = options[:after_render]) && callback.call(env, response)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Rack
|
2
|
+
class Prerender
|
3
|
+
require 'net/http'
|
4
|
+
require_relative 'prerender/constraint'
|
5
|
+
require_relative 'prerender/fetcher'
|
6
|
+
require_relative 'prerender/version'
|
7
|
+
|
8
|
+
attr_reader :app, :constraint, :fetcher
|
9
|
+
|
10
|
+
def initialize(app, options = {})
|
11
|
+
@app = app
|
12
|
+
@constraint = Constraint.new(options)
|
13
|
+
@fetcher = Fetcher.new(options)
|
14
|
+
end
|
15
|
+
|
16
|
+
def call(env)
|
17
|
+
constraint.matches?(env) && fetcher.call(env) || app.call(env)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
metadata
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rack-prerender
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.6.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Todd Hooper
|
8
|
+
- Janosch Müller
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2020-02-14 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rack
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: bundler
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: rake
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: rspec
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: webmock
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
description: Rack middleware to prerender your javascript heavy pages on the fly (fork
|
85
|
+
of prerender_rails).
|
86
|
+
email:
|
87
|
+
- todd@prerender.io
|
88
|
+
- janosch84@gmail.com
|
89
|
+
executables: []
|
90
|
+
extensions: []
|
91
|
+
extra_rdoc_files: []
|
92
|
+
files:
|
93
|
+
- lib/rack/prerender.rb
|
94
|
+
- lib/rack/prerender/constraint.rb
|
95
|
+
- lib/rack/prerender/fetcher.rb
|
96
|
+
- lib/rack/prerender/version.rb
|
97
|
+
homepage: https://github.com/jaynetics/rack-prerender
|
98
|
+
licenses:
|
99
|
+
- MIT
|
100
|
+
metadata: {}
|
101
|
+
post_install_message:
|
102
|
+
rdoc_options: []
|
103
|
+
require_paths:
|
104
|
+
- lib
|
105
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: 2.4.0
|
110
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: '0'
|
115
|
+
requirements: []
|
116
|
+
rubygems_version: 3.1.2
|
117
|
+
signing_key:
|
118
|
+
specification_version: 4
|
119
|
+
summary: Prerender your javascript rendered application on the fly when search engines
|
120
|
+
crawl
|
121
|
+
test_files: []
|