rack-prerender 1.6.2.1 → 1.6.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffd2dd81b8255a205be1cd51795128c9e062eadad3ec443f0a65154ccdaab134
-  data.tar.gz: 8044f53b969b4a6050add6727ff4a83418726defa88125e395bea05c469f5729
+  metadata.gz: e647c21bd7b1376d4fe43b9ca37c8f59b03c46e6d4e8552300666415781cd758
+  data.tar.gz: e4e9bb35c6cd6770438df47cbc8d4b34f28c42c613216b8b4767e26e746d157e
 SHA512:
-  metadata.gz: 918cce61d4efa7e346123a8f2f5a7fe8438517675fd9656624fd1473eb19f2eba59c422b368599b309e21fada596faf9a1c63476607363c1ca54c2853a199dc6
-  data.tar.gz: bcab98539951ba7af5518f4015647ef3e4bb64626b7efa37956f25ca0b9ed6a9c960b67cd46887c22d71f674013aa7dfd8e39adef34c1bffbc659fa9c1331e3a
+  metadata.gz: 5fbcd1f83fb88c38bbd8a402c056dee3493785d192f83e493820f0626cc58ee95fe15b889893901929f024f5675b58ea547175a5c1aa543d20cc3ed25dbe235c
+  data.tar.gz: 452d3a01ecd28f604a5a11765ceb054cf2256b48c31d2c888fcf272e84e3db9c6ad7b527012147d4fd8b0bb4f264c1938fe3eee1b9e9b8dac1a97eceac8c06db

data/lib/rack/prerender/constraint.rb

@@ -10,13 +10,14 @@ module Rack
         @extensions_to_ignore = cast_to_regexp(opts[:extensions_to_ignore] || EXTENSIONS_TO_IGNORE)
       end
 
+      # This is run on every request, so performance matters here.
       def matches?(env)
         return false if env['REQUEST_METHOD'] != 'GET' ||
                         env['HTTP_X_PRERENDER'] ||
                         (user_agent = env['HTTP_USER_AGENT']).nil?
 
         query = env['QUERY_STRING'].to_s
-        return false unless crawler_user_agents.match?(user_agent.downcase) ||
+        return false unless crawler_user_agents.match?(user_agent) ||
                             env['HTTP_X_BUFFERBOT'] ||
                             query.include?('_escaped_fragment_')
 
@@ -25,7 +26,7 @@ module Rack
         return false if extensions_to_ignore.match?(fullpath)
         return false if whitelist && !whitelist.match?(fullpath)
         return false if blacklist && (blacklist.match?(fullpath) ||
-                                      blacklist.match?(env['HTTP_REFERER'].to_s.downcase))
+                                      blacklist.match?(env['HTTP_REFERER'].to_s))
 
         true
       end
@@ -33,14 +34,16 @@ module Rack
       private
 
       def cast_to_regexp(list_arg, escape: true)
-        case list_arg
-        when Regexp, nil
-          list_arg
-        when Array
-          escape ? Regexp.union(list_arg) : Regexp.new(list_arg.join('|'))
-        else
-          Regexp.new(escape ? Regexp.escape(list_arg) : list_arg)
-        end
+        regexp =
+          case list_arg
+          when Regexp, nil
+            list_arg
+          when Array
+            escape ? Regexp.union(list_arg) : Regexp.new(list_arg.join('|'))
+          else
+            Regexp.new(escape ? Regexp.escape(list_arg) : list_arg)
+          end
+        regexp && Regexp.new(regexp.source, Regexp::IGNORECASE)
       end
 
       CRAWLER_USER_AGENTS = [

data/lib/rack/prerender/fetcher.rb

@@ -1,3 +1,5 @@
+require 'zlib'
+
 module Rack
   class Prerender
     class Fetcher
@@ -7,18 +9,31 @@ module Rack
         @options = options
       end
 
+      # Entry point for automatic fetching via the middleware.
       def call(env)
         cached_response = before_render(env)
         return cached_response.finish if cached_response
 
-        if prerendered_response = fetch(env)
+        if prerendered_response = fetch_env(env)
           response = build_rack_response_from_prerender(prerendered_response)
           after_render(env, prerendered_response)
           return response.finish
         end
+
         nil
       end
 
+      # Entry point for manual fetching. Does not run callbacks.
+      def fetch(arg)
+        case arg
+        when String, Symbol, URI then fetch_url(arg.to_s)
+        when Hash                then fetch_env(arg)
+        when Rack::Request       then fetch_env(arg.env)
+        else
+          raise ArgumentError, "expected a URL, Request or env, got #{arg.class}"
+        end
+      end
+
       def before_render(env)
         return unless callback = options[:before_render]
 
@@ -33,60 +48,98 @@ module Rack
         end
       end
 
-      def fetch(env)
-        url = URI.parse(api_url(env))
-        headers = {
-          'User-Agent'      => env['HTTP_USER_AGENT'],
-          'Accept-Encoding' => 'gzip'
-        }
-        headers['X-Prerender-Token'] = ENV['PRERENDER_TOKEN'] if ENV['PRERENDER_TOKEN']
-        headers['X-Prerender-Token'] = options[:prerender_token] if options[:prerender_token]
-        req = Net::HTTP::Get.new(url.request_uri, headers)
-        req.basic_auth(ENV['PRERENDER_USERNAME'], ENV['PRERENDER_PASSWORD']) if options[:basic_auth]
-        http = Net::HTTP.new(url.host, url.port)
-        http.use_ssl = true if url.scheme == 'https'
-        response = http.request(req)
-        if response['Content-Encoding'] == 'gzip'
-          response.body = ActiveSupport::Gzip.decompress(response.body)
-          response['Content-Length'] = response.body.bytesize
-          response.delete('Content-Encoding')
+      def fetch_env(env)
+        fetch_url(request_url(env), as: env['HTTP_USER_AGENT'])
+      end
+
+      def fetch_url(url, as: nil)
+        uri = URI.parse(api_url(url))
+        fetch_api_uri(uri, as: as)
+      end
+
+      # This is just horrible, but replacing net/http would break compatibility
+      # because the response object is leaked to several callbacks :(
+      def fetch_api_uri(uri, as: nil)
+        req = Net::HTTP::Get.new(uri.request_uri, headers(user_agent: as))
+        if options[:basic_auth]
+          req.basic_auth(ENV['PRERENDER_USERNAME'], ENV['PRERENDER_PASSWORD'])
         end
-        response
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true if uri.scheme == 'https'
+        response = http.request(req)
+        decompress(response)
       rescue
         nil
       end
 
-      def api_url(env)
-        new_env = env
-        if env['CF-VISITOR']
-          match = /"scheme":"(http|https)"/.match(env['CF-VISITOR'])
-          new_env['HTTPS'] = true and new_env['rack.url_scheme'] = "https" and new_env['SERVER_PORT'] = 443 if (match && match[1] == 'https')
-          new_env['HTTPS'] = false and new_env['rack.url_scheme'] = "http" and new_env['SERVER_PORT'] = 80 if (match && match[1] == 'http')
+      def api_url(url)
+        if service_url.match?(/[=\/]$/)
+          "#{service_url}#{url}"
+        else
+          "#{service_url}/#{url}"
         end
+      end
 
-        if env['X-FORWARDED-PROTO']
-          new_env['HTTPS'] = true and new_env['rack.url_scheme'] = "https" and new_env['SERVER_PORT'] = 443 if env["X-FORWARDED-PROTO"].split(',')[0] == 'https'
-          new_env['HTTPS'] = false and new_env['rack.url_scheme'] = "http" and new_env['SERVER_PORT'] = 80 if env["X-FORWARDED-PROTO"].split(',')[0] == 'http'
+      def service_url
+        options[:prerender_service_url] || ENV['PRERENDER_SERVICE_URL'] ||
+          'http://service.prerender.io'
+      end
+
+      def headers(user_agent: nil)
+        {
+          'Accept-Encoding'   => 'gzip',
+          'User-Agent'        => user_agent,
+          'X-Prerender-Token' => token,
+        }.compact
+      end
+
+      def token
+        options[:prerender_token] || ENV['PRERENDER_TOKEN']
+      end
+
+      def request_url(env)
+        if env['CF-VISITOR'] && protocol = env['CF-VISITOR'][/"scheme":"(http|https)"/, 1]
+          configure_protocol(env, protocol)
+        end
+
+        if env['X-FORWARDED-PROTO'] && protocol = env["X-FORWARDED-PROTO"].split(',')[0]
+          configure_protocol(env, protocol)
         end
 
-        if options[:protocol]
-          new_env['HTTPS'] = true and new_env['rack.url_scheme'] = "https" and new_env['SERVER_PORT'] = 443 if options[:protocol] == 'https'
-          new_env['HTTPS'] = false and new_env['rack.url_scheme'] = "http" and new_env['SERVER_PORT'] = 80 if options[:protocol] == 'http'
+        if protocol = options[:protocol]
+          configure_protocol(env, protocol)
         end
 
-        url = Rack::Request.new(new_env).url
-        prerender_url = prerender_service_url()
-        forward_slash = prerender_url[-1, 1] == '/' ? '' : '/'
-        "#{prerender_url}#{forward_slash}#{url}"
+        Rack::Request.new(env).url
       end
 
-      def prerender_service_url
-        options[:prerender_service_url] || ENV['PRERENDER_SERVICE_URL'] || 'http://service.prerender.io/'
+      def configure_protocol(env, protocol)
+        return unless protocol == 'http' || protocol == 'https'
+
+        env['rack.url_scheme'] = protocol
+        env['HTTPS']           = protocol == 'https'
+        env['SERVER_PORT']     = protocol == 'https' ? 443 : 80
+      end
+
+      def decompress(response)
+        if response['Content-Encoding'] == 'gzip'
+          response.body =
+            Zlib::GzipReader.wrap(StringIO.new(response.body), &:read)
+          response['Content-Length'] = response.body.bytesize
+          response.delete('Content-Encoding')
+        end
+        response
       end
 
       def build_rack_response_from_prerender(prerendered_response)
-        response = Rack::Response.new(prerendered_response.body, prerendered_response.code, prerendered_response)
-        options[:build_rack_response_from_prerender].call(response, prerendered_response) if options[:build_rack_response_from_prerender]
+        response = Rack::Response.new(
+          prerendered_response.body,
+          prerendered_response.code,
+          prerendered_response,
+        )
+        if callback = options[:build_rack_response_from_prerender]
+          callback.call(response, prerendered_response)
+        end
         response
       end
 

data/lib/rack/prerender/recache_job.rb

@@ -0,0 +1,19 @@
+module Rack
+  class Prerender
+    RecacheJob =
+      if defined?(::ActiveJob::Base)
+        Class.new(::ActiveJob::Base)
+      elsif defined?(::Sidekiq::Worker)
+        Class.new do
+          include ::Sidekiq::Worker
+          instance_eval { alias perform_later perform_async }
+        end
+      else
+        raise NameError, 'requires ActiveJob or Sidekiq'
+      end.class_eval do
+        def perform(url, options)
+          ::Rack::Prerender::Recacher.new(options).call(url)
+        end
+      end
+  end
+end

data/lib/rack/prerender/recacher.rb

@@ -0,0 +1,29 @@
+module Rack
+  class Prerender
+    class Recacher
+      attr_reader :options
+
+      def initialize(options = {})
+        @options = options
+      end
+
+      def call(cached_url)
+        uri = URI(api_url)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+          request = Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json')
+          request.body = %({"prerenderToken":"#{token}","url":"#{cached_url}"})
+          http.request(request) # => Net::HTTPResponse object
+        end
+      end
+
+      def api_url
+        options[:prerender_recache_url] || ENV['PRERENDER_RECACHE_URL'] ||
+          'http://api.prerender.io/recache'
+      end
+
+      def token
+        options[:prerender_token] || ENV['PRERENDER_TOKEN']
+      end
+    end
+  end
+end

data/lib/rack/prerender/version.rb

@@ -2,6 +2,6 @@
 
 module Rack
   class Prerender
-    VERSION = '1.6.2.1'
+    VERSION = '1.6.2.2'
   end
 end

data/lib/rack/prerender.rb

@@ -3,18 +3,36 @@ module Rack
     require 'net/http'
     require_relative 'prerender/constraint'
     require_relative 'prerender/fetcher'
+    require_relative 'prerender/recacher'
     require_relative 'prerender/version'
 
-    attr_reader :app, :constraint, :fetcher
+    attr_accessor :app, :constraint, :fetcher
 
     def initialize(app, options = {})
       @app        = app
       @constraint = Constraint.new(options)
       @fetcher    = Fetcher.new(options)
+      @@options   = options
     end
 
     def call(env)
       constraint.matches?(env) && fetcher.call(env) || app.call(env)
     end
+
+    # utility methods
+
+    def self.fetch(arg, **options)
+      Fetcher.new(@@options.to_h.merge(options)).fetch(arg)
+    end
+
+    def self.recache_now(url, **options)
+      Recacher.new(@@options.to_h.merge(options)).call(url)
+    end
+
+    def self.recache_later(url, **options)
+      # require on demand, so ActiveJob/Sidekiq can come later in load order
+      require_relative 'prerender/recache_job'
+      RecacheJob.perform_later(url, @@options.to_h.merge(options))
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rack-prerender
 version: !ruby/object:Gem::Version
-  version: 1.6.2.1
+  version: 1.6.2.2
 platform: ruby
 authors:
 - Todd Hooper
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-30 00:00:00.000000000 Z
+date: 2020-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -25,6 +25,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -39,6 +53,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -93,6 +121,8 @@ files:
 - lib/rack/prerender.rb
 - lib/rack/prerender/constraint.rb
 - lib/rack/prerender/fetcher.rb
+- lib/rack/prerender/recache_job.rb
+- lib/rack/prerender/recacher.rb
 - lib/rack/prerender/version.rb
 homepage: https://github.com/jaynetics/rack-prerender
 licenses: