seo_cache 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 555ef8fa31e4a10781d79a8366b6b2d72b366c9a7c9760a4be215e937bc5ad47
-  data.tar.gz: 17849429c79a3d5aff80ee63f2d85697ba8cef0b90fdd763dc3fa5c7d63aaabe
+  metadata.gz: 4a029127389aa4942e27c5fbca325e56c020dfe4d35ec743034649e45700b71f
+  data.tar.gz: cb8f7123d7ef7a959e86f53d65c127b4195cfc2b3c21938bf0fb31c6fdcbf507
 SHA512:
-  metadata.gz: 159fcc02649cfafdc96046d64681164e48114907de82802d6ff94207ae3861b94bb4b99410b39291267cd31ca1763d8072480d7096a921bf66327d75ad9f9b39
-  data.tar.gz: de661dfc3d73e00a04834de214b9dc11070899a2ba2647e898a80f0876d92e77d7c2a6442c31483ba263bdb2979a5c10b395d4ce7b1128bc6c2ff498dc8ee3fd
+  metadata.gz: bd0f5a06df0b92a34304fad8893e3b708ef5142bd62509664dba5689fab503ffa1ae7a50901c3ee14f626c92a4d0b3994a017989e3024c8b2e6dd0083cbe47e5
+  data.tar.gz: 7d1e4891181adb3b9deb3cb9c7773b9328befcebfc237f4c76ce3857b6c673751a14c72403126852782726c3b2998fb3253bf7a346d3f4f4f385542b538547bf

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.3.0
+
+- Add more examples in README
+- Switch chromedriver to webdrivers
+- Add tests
+- Set correct URL for Github
+
 ## 0.2.0
 
 - Improve README

data/README.md

@@ -2,6 +2,13 @@
 
 Cache dedicated for SEO with Javascript rendering :fire:
 
+## Purpose
+
+Google credo is: Don't waste my bot time!
+
+So to reduce Googlebot crawling time, let's provide HTML files in a specific cache.
+
+This cache is suitable for static (generated or not) pages but not for connected pages.
 
 ## Installation
 
@@ -19,42 +26,98 @@ Or install it yourself as:
 
     $ gem install seo_cache
 
-Install chrome driver on your device
+Install chromium or chrome driver on your device (the chromedriver will be automatically downloaded).
 
-## How it works
+Declare the middleware. For instance in `config/initializers/seo_cache.rb`:
 
-Specific cache for bots to optimize time to first byte and render Javascript on server side.
+```ruby
+require 'seo_cache'
+
+# See options below
 
-Options:
+Rails.application.config.middleware.use SeoCache::Middleware
+```
 
-Choose a cache mode (`disk` or `memory`):
+## Options
+
+Chrome path (**required**) (`disk` or `memory`):
+
+    SeoCache.chrome_path = Rails.env.development? ? '/usr/bin/chromium-browser' : '/usr/bin/chromium'
+
+Choose a cache mode (`memory` (default) or `disk`):
 
     SeoCache.cache_mode = 'memory'
 
-If cache on disk, specify the cache path (e.g. `Rails.root.join('public', 'seo_cache')`):
+Disk cache path (required if disk cache):
+    
+    SeoCache.disk_cache_path = Rails.root.join('public', 'seo_cache')
+
+Redis URL (required if memory cache):
     
-    SeoCache.disk_cache_path = nil
+    SeoCache.redis_url = "redis://localhost:6379/"
+
+Redis prefix:
+    
+    SeoCache.redis_namespace = '_my_project:seo_cache'
+
+Specific log file (if you want to log missed cache urls):
+    
+    SeoCache.logger_path = Rails.root.join('log', 'seo_cache.log')
+
+Activate missed cache urls:
+    
+    SeoCache.log_missed_cache = true
     
 URLs to blacklist:
 
-    SeoCache.blacklist_urls = []
+    SeoCache.blacklist_params = %w[^/assets/.* ^/admin.*]
+    
+Params to blacklist:
+
+    SeoCache.blacklist_urls = %w[page]
     
 URLs to whitelist:
 
     SeoCache.whitelist_urls = []
     
-Query params un URl to blacklist:
+Parameter to add manually to the URl to force page caching, if you want to cache a specific URL (e.g. `https://<my_website>/?_seo_cache_=true`):
+
+    SeoCache.force_cache_url_param = '_seo_cache_'
+    
+URL extension to ignore when caching (already defined):
+
+    SeoCache.extensions_to_ignore = [<your_list>]
+    
+List of bot agents (already defined):
+
+    SeoCache.crawler_user_agents = [<your_list>]
+    
+Parameter added to URL when generating the page, avoid infinite rendering (override only if already used):
 
-    SeoCache.blacklist_params = []
+    SeoCache.prerender_url_param = '_prerender_'
+
+Be aware, JS will be render twice: once by server rendering and once by client. For React, this not a problem but with jQuery plugins, it can duplicate elements in the page (you have to check the redundancy). 
 
 ## Automatic caching
 
-To automate cache, create a cron rake task which called:
+To automate caching, create a cron rake task (e.g. in `lib/tasks/populate_seo_cache.rake`):
 
 ```ruby
-SeoCache::PopulateCache.new('https://<your-domain-name>', paths_to_cache).new.perform
+namespace :MyProject do
+
+  desc 'Populate cache for SEO'
+  task populate_seo_cache: :environment do |_task, _args|
+    require 'seo_cache/populate_cache'
+    
+    paths_to_cache = public_paths_like_sitemap
+    
+    SeoCache::PopulateCache.new('https://<your-domain-name>', paths_to_cache).new.perform
+  end
+end
 ```
 
+You can add the `force_cache: true` option to `SeoCache::PopulateCache` for overwrite cache data.
+
 ## Server
 
 If you use disk caching, add to your Nginx configuration:
@@ -89,6 +152,78 @@ location / {
 }
 ```
 
+## Heroku case
+
+If you use Heroku server, you can't store file on dynos. But you have two alternatives :
+
+- Use the memory mode
+
+- Use a second server (a dedicated one) to store HTML files and combine with Nginx.
+
+To intercept the request, use the following middleware in Rails:
+
+In `config/initializers`, create a new file:
+
+```ruby
+require 'bot_detector'
+
+if Rails.env.production?
+  Rails.application.config.middleware.insert_before ActionDispatch::Static, BotDetector
+end
+``` 
+
+Then in `lib` directory, for instance, manage the request:
+
+```ruby
+class BotRedirector
+  CRAWLER_USER_AGENTS = ['googlebot', 'yahoo', 'bingbot', 'baiduspider', 'facebookexternalhit', 'twitterbot', 'rogerbot', 'linkedinbot', 'embedly', 'bufferbot', 'quora link preview', 'showyoubot', 'outbrain', 'pinterest/0.', 'developers.google.com/+/web/snippet', 'www.google.com/webmasters/tools/richsnippets', 'slackbot', 'vkShare', 'W3C_Validator', 'redditbot', 'Applebot', 'WhatsApp', 'flipboard', 'tumblr', 'bitlybot', 'SkypeUriPreview', 'nuzzel', 'Discordbot', 'Google Page Speed', 'Qwantify'].freeze
+
+  IGNORE_URLS = [
+    '/robots.txt'
+  ].freeze
+
+  def initialize(app)
+    @app = app
+  end
+
+  def call(env)
+    if env['HTTP_USER_AGENT'].present? && CRAWLER_USER_AGENTS.any? { |crawler_user_agent| env['HTTP_USER_AGENT'].downcase.include?(crawler_user_agent.downcase) }
+      begin
+        request = Rack::Request.new(env)
+
+        return @app.call(env) if IGNORE_URLS.any? { |ignore_url| request.fullpath.downcase =~ /^#{ignore_url.downcase}/ }
+
+        url     = URI.parse(ENV['SEO_SERVER'] + request.fullpath)
+        headers = {
+          'User-Agent'      => env['HTTP_USER_AGENT'],
+          'Accept-Encoding' => 'gzip'
+        }
+        req     = Net::HTTP::Get.new(url.request_uri, headers)
+        # req.basic_auth(ENV['SEO_USER_ID'], ENV['SEO_PASSWD']) # if authentication mechanism
+        http         = Net::HTTP.new(url.host, url.port)
+        http.use_ssl = true if url.scheme == 'https'
+        response     = http.request(req)
+        if response['Content-Encoding'] == 'gzip'
+          response.body              = ActiveSupport::Gzip.decompress(response.body)
+          response['Content-Length'] = response.body.length
+          response.delete('Content-Encoding')
+        end
+
+        return [response.code.to_i, { 'Content-Type' => response.header['Content-Type'] }, [response.body]]
+      rescue => error
+        Rails.logger.error("[bot_redirection] #{error.message}")
+
+        @app.call(env)
+      end
+    else
+      @app.call(env)
+    end
+  end
+end
+```
+
+If you use a second server, all links must be relatives in your HTML files, to avoid multi-domains links.
+
 ## Inspiration
 
 Inspired by [prerender gem](https://github.com/prerender/prerender_rails).

data/lib/seo_cache/logger.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'logger'
+
+module SeoCache
+  #
+  # @example Enable full logging
+  #   SeoCache.logger.level = :debug
+  #
+  # @example Log to file
+  #   SeoCache.logger.output = 'seo_cache.log'
+  #
+  # @example Use logger manually
+  #   SeoCache.logger.info('This is info message')
+  #   SeoCache.logger.warn('This is warning message')
+  #
+  class Logger
+    extend Forwardable
+    include ::Logger::Severity
+
+    def_delegators :@logger, :debug, :debug?,
+                   :info, :info?,
+                   :warn, :warn?,
+                   :error, :error?,
+                   :fatal, :fatal?,
+                   :level
+
+    def initialize(logger_path = nil)
+      @logger = create_logger(logger_path || $stdout)
+    end
+
+    def output=(io)
+      # `Logger#reopen` was added in Ruby 2.3
+      if @logger.respond_to?(:reopen)
+        @logger.reopen(io)
+      else
+        @logger = create_logger(io)
+      end
+    end
+
+    #
+    # For Ruby < 2.3 compatibility
+    # Based on https://github.com/ruby/ruby/blob/ruby_2_3/lib/logger.rb#L250
+    #
+
+    def level=(severity)
+      if severity.is_a?(Integer)
+        @logger.level = severity
+      else
+        case severity.to_s.downcase
+        when 'debug'.freeze
+          @logger.level = DEBUG
+        when 'info'.freeze
+          @logger.level = INFO
+        when 'warn'.freeze
+          @logger.level = WARN
+        when 'error'.freeze
+          @logger.level = ERROR
+        when 'fatal'.freeze
+          @logger.level = FATAL
+        when 'unknown'.freeze
+          @logger.level = UNKNOWN
+        else
+          raise ArgumentError, "invalid log level: #{severity}"
+        end
+      end
+    end
+
+    #
+    # Returns IO object used by logger internally.
+    #
+    # Normally, we would have never needed it, but we want to
+    # use it as IO object for all child processes to ensure their
+    # output is redirected there.
+    #
+    # It is only used in debug level, in other cases output is suppressed.
+    #
+    # @api private
+    #
+    def io
+      @logger.instance_variable_get(:@logdev).instance_variable_get(:@dev)
+    end
+
+    #
+    # Marks code as deprecated with replacement.
+    #
+    # @param [String] old
+    # @param [String] new
+    #
+    def deprecate(old, new)
+      warn "[DEPRECATION] #{old} is deprecated. Use #{new} instead."
+    end
+
+    private
+
+    def create_logger(output)
+      logger           = ::Logger.new(output)
+      logger.progname  = 'SeoCache'
+      logger.level     = ($DEBUG ? DEBUG : INFO)
+      logger.formatter = proc do |severity, time, progname, msg|
+        "#{time.strftime('%F %T')} #{severity} #{progname} #{msg}\n"
+      end
+
+      logger
+    end
+  end
+end

data/lib/seo_cache/page_caching.rb

@@ -35,7 +35,7 @@ module SeoCache
 
     def cache_exists?(path)
       if SeoCache.memory_cache? && @redis
-        @redis.exists?(cache_path(path))
+        @redis.exists(cache_path(path))
       else
         File.exist?(cache_path(path))
       end

data/lib/seo_cache/page_render.rb

@@ -33,6 +33,8 @@ module SeoCache
     def init_driver
       # Selenium::WebDriver.logger.level = :info
 
+      Selenium::WebDriver::Chrome.path = SeoCache.chrome_path if SeoCache.chrome_path
+
       client          = ::Selenium::WebDriver::Remote::Http::Persistent.new
       browser_options = ::Selenium::WebDriver::Chrome::Options.new
       browser_options.args << '--headless'

data/lib/seo_cache/version.rb

@@ -1,3 +1,3 @@
 module SeoCache
-  VERSION = '0.2.0'.freeze
+  VERSION = '0.3.0'.freeze
 end

data/lib/seo_cache.rb

@@ -1,23 +1,27 @@
 # frozen_string_literal: true
 
-require 'net/http'
 require 'active_support'
-require 'selenium/webdriver'
-require 'chromedriver-helper'
-require 'selenium/webdriver/remote/http/persistent'
+require 'net/http'
 require 'redis'
 require 'redis-namespace'
+require 'selenium/webdriver'
+require 'selenium/webdriver/remote/http/persistent'
+require 'webdrivers'
 
+require 'seo_cache/logger'
 require 'seo_cache/version'
 require 'seo_cache/middleware'
 
 module SeoCache
 
+  mattr_accessor :chrome_path
+  self.chrome_path = nil
+
   mattr_accessor :cache_mode # disk or memory
   self.cache_mode = 'memory'
 
   mattr_accessor :disk_cache_path
-  self.disk_cache_path = nil
+  self.disk_cache_path = ''
 
   mattr_accessor :disk_cache_extension
   self.disk_cache_extension = '.html'
@@ -89,6 +93,15 @@ module SeoCache
     'Qwantify'
   ]
 
+  mattr_accessor :logger_path
+  self.logger_path = nil
+
+  mattr_accessor :logger_level
+  self.logger_level = :INFO
+
+  mattr_accessor :logger
+  # self.logger = SeoCache::Logger.new(SeoCache.logger_path)
+
   def self.memory_cache?
     SeoCache.cache_mode == 'memory'
   end
@@ -97,11 +110,17 @@ module SeoCache
     SeoCache.cache_mode == 'disk'
   end
 
+  def self.logger
+    @logger ||= SeoCache::Logger.new(SeoCache.logger_path)
+  end
+
   def self.log(message)
-    Rails.logger.info { "[seo_cache] #{message}" }
+    SeoCache.logger.info(message)
+    # Rails.logger.info { "[seo_cache] #{message}" }
   end
 
   def self.log_error(message)
-    Rails.logger.error { "[seo_cache] #{message}" }
+    SeoCache.logger.error(message)
+    # Rails.logger.error { "[seo_cache] #{message}" }
   end
 end

data/seo_cache.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
 
   spec.summary     = 'Cache dedicated for SEO with Javascript rendering'
   spec.description = 'Specific cache for bots to optimize time to first byte and render Javascript on server side.'
-  spec.homepage    = 'https://github.com/floXcoder/SeoCache'
+  spec.homepage    = 'https://github.com/floXcoder/seo_cache'
   spec.license     = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
@@ -20,16 +20,18 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'activesupport', '~> 5'
+  spec.add_dependency 'net-http-persistent', '~> 3'
   spec.add_dependency 'rack', '~> 2'
   spec.add_dependency 'railties', '~> 5'
-  spec.add_dependency 'net-http-persistent', '~> 3'
-  spec.add_dependency 'activesupport', '~> 5'
-  spec.add_dependency 'selenium-webdriver', '~> 3'
-  spec.add_dependency 'chromedriver-helper', '~> 2'
   spec.add_dependency 'redis', '~> 4'
   spec.add_dependency 'redis-namespace', '~> 1'
+  spec.add_dependency 'selenium-webdriver', '~> 3'
+  spec.add_dependency 'webdrivers', '~> 3'
 
   spec.add_development_dependency 'bundler', '~> 1'
   spec.add_development_dependency 'rake', '~> 12'
   spec.add_development_dependency 'rspec', '~> 3'
+  spec.add_development_dependency 'simplecov', '~> 0.16'
+  spec.add_development_dependency 'webmock', '~> 3'
 end

metadata

@@ -1,59 +1,59 @@
 --- !ruby/object:Gem::Specification
 name: seo_cache
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - FloXcoder
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-26 00:00:00.000000000 Z
+date: 2019-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rack
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '5'
 - !ruby/object:Gem::Dependency
-  name: railties
+  name: net-http-persistent
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '3'
 - !ruby/object:Gem::Dependency
-  name: net-http-persistent
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '2'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: railties
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -67,61 +67,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '5'
 - !ruby/object:Gem::Dependency
-  name: selenium-webdriver
+  name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '4'
 - !ruby/object:Gem::Dependency
-  name: chromedriver-helper
+  name: redis-namespace
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1'
 - !ruby/object:Gem::Dependency
-  name: redis
+  name: selenium-webdriver
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '3'
 - !ruby/object:Gem::Dependency
-  name: redis-namespace
+  name: webdrivers
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +164,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
 description: Specific cache for bots to optimize time to first byte and render Javascript
   on server side.
 email:
@@ -186,13 +214,14 @@ files:
 - bin/console
 - bin/setup
 - lib/seo_cache.rb
+- lib/seo_cache/logger.rb
 - lib/seo_cache/middleware.rb
 - lib/seo_cache/page_caching.rb
 - lib/seo_cache/page_render.rb
 - lib/seo_cache/populate_cache.rb
 - lib/seo_cache/version.rb
 - seo_cache.gemspec
-homepage: https://github.com/floXcoder/SeoCache
+homepage: https://github.com/floXcoder/seo_cache
 licenses:
 - MIT
 metadata: {}