seo_cache 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30638911700f93015a1ebe83773cc0e88c5cbf8682a3951481f9bc5fa733e00d
-  data.tar.gz: c2d4cf35ef1389992fb92465b31ccb83b174bad535438e839dabaa81dbcc3ca1
+  metadata.gz: e101c3d2b9260ac681e1d6ffc399c88e98c5f1edc69f3148790e6b63abda2ad0
+  data.tar.gz: de86f2455bb55ecbbf70365c068f6ba3b4f865cb59dfb352975a6f14d4599681
 SHA512:
-  metadata.gz: 919ca5401c7f11763324cc851867944e920569c41533092a0767841bcf94a189539772373be819d4cf14400634a30c05f0fa8edfd26ab63365ecd9da610446bf
-  data.tar.gz: 5209689caeeb22a54c3ff4a3dfe5d8eae843f8d02d5322e5e6e94400978cf564a910d7d7cfdc7dfe4f7b712824e5d4ea925bd3ac0b781bbdc32c0f95769c3e74
+  metadata.gz: 2539f235f5e806422cbd969723a3a83cb671b270869ec02f73efffa2d15f96bf401155bcc48ea462ad1ee7fac2b994692b94103decd33ec61f5ee88672c5ad16
+  data.tar.gz: d871e975c11792c10bde694427e184e19d1b7effa0ff5eb90bd88daa05ed1b42a66deccbe111e2435fb0972df24556a88c462956eb47f59d4c1875fe94e66be2

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.9.0
+
+- Update gems
+- Don't cache response if status is not 200
+- Add 'seo_mode' variable to env to detect si seo mode is currently active
+- Add status to source code to cache only pages with 200 HTTP status code
+- Check existence of destination directory before creating it
+- Remove persistent connection (already built-in in Selenium) 
+
 ## 0.8.0
 
 - Change bundler version

data/README.md

@@ -112,6 +112,12 @@ Parameter to add manually to the URl to force page caching, if you want to cache
 SeoCache.force_cache_url_param = '_seo_cache_'
 ```
 
+Cache only the pages with these HTTP status code (don't cache by default not found or error pages):
+
+```ruby
+SeoCache.cache_only_status = [<your_list>]
+```
+
 URL extension to ignore when caching (already defined):
 
 ```ruby
@@ -134,6 +140,15 @@ Be aware, JS will be render twice: once by server rendering and once by client.
 
 Disk cache is recommended by default. Nginx will directly fetch file on disk. The TTFB (time to first byte) will be under 200ms :). You can use memory cache if you have lot of RAM.
 
+## Controllers
+
+You can check if seo mode is active in your controllers, with the following variable:
+
+```ruby
+request.env['seo_mode']
+```
+
+
 ## Consult cache pages
 
 Too see in browser the cache page, open a browser and set the user agent to:

data/lib/seo_cache/middleware.rb

@@ -6,9 +6,9 @@ require 'seo_cache/page_render'
 module SeoCache
   class Middleware
     def initialize(app, options = {})
-      @options                  = options
-      @extensions_to_ignore     = SeoCache.extensions_to_ignore
-      @crawler_user_agents      = SeoCache.crawler_user_agents
+      @options              = options
+      @extensions_to_ignore = SeoCache.extensions_to_ignore
+      @crawler_user_agents  = SeoCache.crawler_user_agents
 
       @app = app
 
@@ -16,7 +16,7 @@ module SeoCache
     end
 
     def call(env)
-      if should_show_prerendered_page(env)
+      if prerender_page?(env)
         cached_response = before_render(env)
 
         return cached_response.finish if cached_response.present?
@@ -24,40 +24,61 @@ module SeoCache
         SeoCache.log('missed cache : ' + Rack::Request.new(env).path) if SeoCache.log_missed_cache
 
         if SeoCache.prerender_service_url.present?
-          prerendered_response = get_prerendered_page_response(env)
-          if prerendered_response
-            response = build_rack_response_from_prerender(prerendered_response.body)
-            after_render(env, prerendered_response)
+          prerender_response = prerender_service(env)
+          if prerender_response
+            response = build_response_from_prerender(prerender_response.body)
+            after_render(env, prerender_response)
             return response.finish
           end
         else
           Thread.new do
-            prerendered_data = page_render(env)
-            after_render(env, prerendered_data) if prerendered_data
+            prerender_data = page_render(env)
+            # Extract status from render page
+            status = prerender_data.scan(/<!--status:(\d+)-->/).last&.first
+            after_render(env, prerender_data, status || 200)
           end
         end
+      elsif prerender_params?(env)
+        env['seo_mode'] = true
+        # Add status to render page because Selenium doesn't return http headers or status...
+        status, headers, response = @app.call(env)
+        status_code               = "<!--status:#{status}-->"
+        # Cannot add at the top of file, Chrome removes leading comments...
+        body_code = response.body.sub('<head>', "<head>#{status_code}")
+        return [status, headers, [body_code]]
       end
 
-      @app.call(env)
+      return @app.call(env)
     end
 
-    def should_show_prerendered_page(env)
-      user_agent                     = env['HTTP_USER_AGENT']
-      buffer_agent                   = env['HTTP_X_BUFFERBOT']
-      is_requesting_prerendered_page = false
+    def prerender_params?(env)
+      return false if env['REQUEST_METHOD'] != 'GET'
+
+      request      = Rack::Request.new(env)
+      query_params = Rack::Utils.parse_query(request.query_string)
+
+      return false if @extensions_to_ignore.any? { |extension| request.fullpath.include? extension }
+
+      return true if query_params.has_key?(SeoCache.prerender_url_param) || query_params.has_key?(SeoCache.force_cache_url_param)
+    end
+
+    def prerender_page?(env)
+      user_agent                   = env['HTTP_USER_AGENT']
+      buffer_agent                 = env['HTTP_X_BUFFERBOT']
+      is_requesting_prerender_page = false
 
       return false unless user_agent
 
       return false if env['REQUEST_METHOD'] != 'GET'
 
-      request = Rack::Request.new(env)
+      request      = Rack::Request.new(env)
       query_params = Rack::Utils.parse_query(request.query_string)
 
       # If it is the generated page...don't prerender
       return false if query_params.has_key?(SeoCache.prerender_url_param)
 
       # if it is a bot and host doesn't contain these domains...don't prerender
-      return false if SeoCache.whitelist_hosts.present? && SeoCache.whitelist_hosts.any? { |host| !request.host.include?(host) }
+      return false if SeoCache.whitelist_hosts.present? && SeoCache.whitelist_hosts.none? { |host| request.host.include?(host) }
 
       # if it is a bot and urls contain these params...don't prerender
       return false if SeoCache.blacklist_params.present? && SeoCache.blacklist_params.any? { |param| query_params.has_key?(param) }
@@ -79,20 +100,20 @@ module SeoCache
       end
       return false if blacklisted_url
 
-      is_requesting_prerendered_page = true if Rack::Utils.parse_query(request.query_string).has_key?('_escaped_fragment_') || Rack::Utils.parse_query(request.query_string).has_key?(SeoCache.force_cache_url_param)
+      is_requesting_prerender_page = true if Rack::Utils.parse_query(request.query_string).has_key?('_escaped_fragment_') || Rack::Utils.parse_query(request.query_string).has_key?(SeoCache.force_cache_url_param)
 
       # if it is a bot...show prerendered page
-      is_requesting_prerendered_page = true if @crawler_user_agents.any? { |crawler_user_agent| user_agent.downcase.include?(crawler_user_agent.downcase) }
+      is_requesting_prerender_page = true if @crawler_user_agents.any? { |crawler_user_agent| user_agent.downcase.include?(crawler_user_agent.downcase) }
 
       # if it is BufferBot...show prerendered page
-      is_requesting_prerendered_page = true if buffer_agent
+      is_requesting_prerender_page = true if buffer_agent
 
       SeoCache.log('force cache : ' + request.path) if Rack::Utils.parse_query(request.query_string).has_key?(SeoCache.force_cache_url_param) && SeoCache.log_missed_cache
 
-      return is_requesting_prerendered_page
+      return is_requesting_prerender_page
     end
 
-    def get_prerendered_page_response(env)
+    def prerender_service(env)
       url          = URI.parse(build_api_url(env))
       headers      = {
         'User-Agent'      => env['HTTP_USER_AGENT'],
@@ -107,7 +128,8 @@ module SeoCache
         response['Content-Length'] = response.body.length
         response.delete('Content-Encoding')
       end
-      response
+
+      return response
     rescue StandardError => error
       SeoCache.log_error(error.message)
     end
@@ -136,8 +158,8 @@ module SeoCache
       "#{prerender_url}#{forward_slash}#{url}"
     end
 
-    def build_rack_response_from_prerender(prerendered_response)
-      response = Rack::Response.new(prerendered_response.body, prerendered_response.code, prerendered_response.header)
+    def build_response_from_prerender(prerender_response)
+      response = Rack::Response.new(prerender_response.body, prerender_response.code, prerender_response.header)
 
       # @options[:build_rack_response_from_prerender]&.call(response, prerendered_response)
 
@@ -152,17 +174,14 @@ module SeoCache
 
       return nil unless cached_render
 
-      if cached_render&.is_a?(String)
+      if cached_render.is_a?(String)
         Rack::Response.new(cached_render, 200, 'Content-Type' => 'text/html; charset=utf-8')
-      elsif cached_render&.is_a?(Rack::Response)
+      elsif cached_render.is_a?(Rack::Response)
         cached_render
       end
     end
 
     def page_render(env)
-      # return nil unless @options[:page_render]
-      # @options[:page_render].call(url)
-
       # Add key parameter to url
       request = Rack::Request.new(env)
       url     = if request.query_string.present? || request.url.end_with?('?')
@@ -172,12 +191,11 @@ module SeoCache
                 end
       url     += "#{SeoCache.prerender_url_param}=true"
 
-      PageRender.new.get(url)
+      return PageRender.new.get(url)
     end
 
-    def after_render(env, response)
-      # return true unless @options[:after_render]
-      # @options[:after_render].call(env, response)
+    def after_render(env, response, status = 200)
+      return unless response && SeoCache.cache_only_status.include?(status.to_i)
 
       @page_caching.cache(response, Rack::Request.new(env).path)
     end

data/lib/seo_cache/page_caching.rb

@@ -74,7 +74,7 @@ module SeoCache
     end
 
     def write_to_disk(content, path, gzip)
-      FileUtils.makedirs(File.dirname(path))
+      FileUtils.makedirs(File.dirname(path)) unless File.directory?(File.dirname(path))
       File.open(path, 'wb+') { |f| f.write(content) }
 
       Zlib::GzipWriter.open(path + '.gz', gzip) { |f| f.write(content) } if gzip

data/lib/seo_cache/page_render.rb

@@ -12,16 +12,8 @@ module SeoCache
       return @driver.page_source
     rescue StandardError => error
       SeoCache.log_error(error.message)
-    ensure
-      @driver&.quit
-    end
-
-    def persistent_get(url)
-      @driver.get(url)
-
-      return @driver.page_source
-    rescue StandardError => error
-      SeoCache.log_error(error.message)
+    # ensure
+    #   @driver&.quit
     end
 
     def close_connection
@@ -37,7 +29,6 @@ module SeoCache
 
       Selenium::WebDriver::Chrome.path = SeoCache.chrome_path if SeoCache.chrome_path
 
-      client          = ::Selenium::WebDriver::Remote::Http::Persistent.new
       browser_options = ::Selenium::WebDriver::Chrome::Options.new
       browser_options.args << '--headless'
       browser_options.args << '--disable-gpu'
@@ -45,7 +36,7 @@ module SeoCache
       browser_options.args << '--disable-web-security'
       browser_options.args << '--window-size=1920x1080'
       # browser_options.args << '--remote-debugging-port=3020'
-      @driver = ::Selenium::WebDriver.for(:chrome, options: browser_options, http_client: client)
+      @driver = ::Selenium::WebDriver.for(:chrome, options: browser_options)
     end
   end
 end

data/lib/seo_cache/populate_cache.rb

@@ -18,7 +18,7 @@ module SeoCache
       @paths.each do |path|
         next if @page_caching.cache_exists?(path) && !@force_cache
 
-        page_source = @page_render.persistent_get(@host + path)
+        page_source = @page_render.get(@host + path)
         @page_caching.cache(page_source, path)
       end
 

data/lib/seo_cache/version.rb

@@ -1,3 +1,3 @@
 module SeoCache
-  VERSION = '0.8.0'.freeze
+  VERSION = '0.9.0'.freeze
 end

data/lib/seo_cache.rb

@@ -5,7 +5,6 @@ require 'net/http'
 require 'redis'
 require 'redis-namespace'
 require 'selenium/webdriver'
-require 'selenium/webdriver/remote/http/persistent'
 require 'webdrivers'
 
 require 'seo_cache/logger'
@@ -53,14 +52,18 @@ module SeoCache
   mattr_accessor :prerender_service_url
   self.prerender_service_url = nil
 
+  # Selenium doesn't support HTTP header, so URL parameter is mandatory
   mattr_accessor :prerender_url_param
   self.prerender_url_param = '_prerender_'
 
   mattr_accessor :force_cache_url_param
   self.force_cache_url_param = '_seo_cache_'
 
+  mattr_accessor :cache_only_status
+  self.cache_only_status = [200]
+
   mattr_accessor :extensions_to_ignore
-  self.extensions_to_ignore = %w[.js .css .xml .less .png .jpg .jpeg .gif .pdf .doc .txt .ico .rss .zip .mp3 .rar .exe .wmv .doc .avi .ppt .mpg .mpeg .tif .wav .mov .psd .ai .xls .mp4 .m4a .swf .dat .dmg .iso .flv .m4v .torrent]
+  self.extensions_to_ignore = %w[.js .css .xml .less .png .jpg .jpeg .gif .pdf .doc .txt .ico .rss .zip .mp3 .rar .exe .wmv .doc .avi .ppt .mpg .mpeg .tif .wav .mov .psd .ai .xls .mp4 .m4a .swf .dat .dmg .iso .flv .m4v .torrent .woff2 .woff .gz .ttf .svg]
 
   mattr_accessor :crawler_user_agents
   self.crawler_user_agents = [

data/seo_cache.gemspec

@@ -21,12 +21,11 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activesupport', '~> 5'
-  spec.add_dependency 'net-http-persistent', '~> 3'
   spec.add_dependency 'rack', '~> 2'
   spec.add_dependency 'railties', '~> 5'
   spec.add_dependency 'redis', '~> 4'
   spec.add_dependency 'redis-namespace', '~> 1'
-  spec.add_dependency 'selenium-webdriver', '3.141.0' # 3.142 is bugged
+  spec.add_dependency 'selenium-webdriver', '~> 3'
   spec.add_dependency 'webdrivers', '~> 4'
 
   spec.add_development_dependency 'bundler', '~> 1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: seo_cache
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - FloXcoder
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-10 00:00:00.000000000 Z
+date: 2019-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -24,20 +24,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5'
-- !ruby/object:Gem::Dependency
-  name: net-http-persistent
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -98,16 +84,16 @@ dependencies:
   name: selenium-webdriver
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.141.0
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.141.0
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: webdrivers
   requirement: !ruby/object:Gem::Requirement
@@ -240,8 +226,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.9
+rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
 summary: Cache dedicated for SEO with Javascript rendering