onebox 2.2.8 → 2.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5899faff4b0c2e96d766892a810e866e28a188dbd7b9e8b2c7bdd6bc5a7d7490
-  data.tar.gz: 7d739d48349f32dc3f0675d85b29446d6f758f276b845543cc9f9d359c9eb664
+  metadata.gz: 541807b758c73492e2868271a63b8f05ef4d94aad316be86fb5c42c3c3f0e2ce
+  data.tar.gz: 10f460d3d6a1b09a6087a38ad0692c9f652e027beb65b22845536480055b4a48
 SHA512:
-  metadata.gz: 790964df42c83ea05ff052727a7ad4b567f6064be1a25bf974eeac661a6f7daa3696c64686f21bb83df30ad280caffc36b2039e3c0b799a3171713d61c88eff2
-  data.tar.gz: de30b5e5e6110fd72c0aa792aebffa7571c50f04604efedac648e754a527f4945a784328eb43ce0352e9b06eeac908ea23ff0dcdf3ce4d84ecb5e8c3dd64dfd6
+  metadata.gz: d13068e9dbb437388ee689581bc64bb08bf4c5ae898755680c3c978a00fc4a4ee13004f0f2a3071f40d226d15e84f799a949d0d07572c6537ddd92950416d56b
+  data.tar.gz: 1efacb58ff5d24daff63cdd855a32ae80826f3585eee0305ba032269ee4431f1541848582bd5587aacffdcd6cb9c5bb35c650706c6028ed5e115e17daf362621

data/lib/onebox/engine/amazon_onebox.rb

@@ -11,11 +11,22 @@ module Onebox
       include HTML
 
       always_https
-      matches_regexp(/^https?:\/\/(?:www\.)?(?:smile\.)?(amazon|amzn)\.(?<tld>com|ca|de|it|es|fr|co\.jp|co\.uk|cn|in|com\.br|com\.mx)\//)
+      matches_regexp(/^https?:\/\/(?:www\.)?(?:smile\.)?(amazon|amzn)\.(?<tld>com|ca|de|it|es|fr|co\.jp|co\.uk|cn|in|com\.br|com\.mx|nl|pl|sa|sg|se|com\.tr|ae)\//)
 
       def url
+        # Have we cached the HTML body of the requested URL?
+        # If so, try to grab the canonical URL from that document,
+        # rather than guess at the best URL structure to use
+        if @body_cacher && @body_cacher.respond_to?('cache_response_body?')
+          if @body_cacher.cached_response_body_exists?(uri.to_s)
+            @raw ||= Onebox::Helpers.fetch_html_doc(@url, http_params, @body_cacher)
+            canonical_link = @raw.at('//link[@rel="canonical"]/@href')
+            return canonical_link.to_s if canonical_link
+          end
+        end
+
         if match && match[:id]
-          return "https://www.amazon.#{tld}/gp/aw/d/#{Onebox::Helpers.uri_encode(match[:id])}"
+          return "https://www.amazon.#{tld}/dp/#{Onebox::Helpers.uri_encode(match[:id])}"
         end
 
         @url
@@ -26,10 +37,9 @@ module Onebox
       end
 
       def http_params
-        {
-          'User-Agent' =>
-          'Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'
-        }
+        if @options && @options[:user_agent]
+          { 'User-Agent' => @options[:user_agent] }
+        end
       end
 
       private

data/lib/onebox/engine/gfycat_onebox.rb

@@ -63,7 +63,7 @@ module Onebox
 
       def nokogiri_page
         @nokogiri_page ||= begin
-          response = Onebox::Helpers.fetch_response(url, 10) rescue nil
+          response = Onebox::Helpers.fetch_response(url, redirect_limit: 10) rescue nil
           Nokogiri::HTML(response)
         end
       end

data/lib/onebox/engine/google_docs_onebox.rb

@@ -47,7 +47,7 @@ module Onebox
       end
 
       def get_og_data
-        response = Onebox::Helpers.fetch_response(url, 10) rescue nil
+        response = Onebox::Helpers.fetch_response(url, redirect_limit: 10) rescue nil
         html = Nokogiri::HTML(response)
         og_data = {}
         html.css('meta').each do |m|

data/lib/onebox/engine/html.rb

@@ -11,7 +11,8 @@ module Onebox
       end
 
       def raw
-        @raw ||= Onebox::Helpers.fetch_html_doc(url, http_params)
+        body_cacher = self.options[:body_cacher] if self.options
+        @raw ||= Onebox::Helpers.fetch_html_doc(url, http_params, body_cacher)
       end
 
       def html?

data/lib/onebox/engine/pastebin_onebox.rb

@@ -31,7 +31,7 @@ module Onebox
 
       def lines
         return @lines if @lines
-        response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1) rescue ""
+        response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", redirect_limit: 1) rescue ""
         @lines = response.split("\n")
       end
 

data/lib/onebox/engine/twitter_status_onebox.rb

@@ -17,7 +17,7 @@ module Onebox
       private
 
       def get_twitter_data
-        response = Onebox::Helpers.fetch_response(url, nil, nil, http_params) rescue nil
+        response = Onebox::Helpers.fetch_response(url, headers: http_params) rescue nil
         html = Nokogiri::HTML(response)
         twitter_data = {}
         html.css('meta').each do |m|

data/lib/onebox/helpers.rb

@@ -24,8 +24,8 @@ module Onebox
       html.gsub(/<[^>]+>/, ' ').gsub(/\n/, '')
     end
 
-    def self.fetch_html_doc(url, headers = nil)
-      response = (fetch_response(url, nil, nil, headers) rescue nil)
+    def self.fetch_html_doc(url, headers = nil, body_cacher = nil)
+      response = (fetch_response(url, headers: headers, body_cacher: body_cacher) rescue nil)
       doc = Nokogiri::HTML(response)
       uri = Addressable::URI.parse(url)
 
@@ -37,7 +37,7 @@ module Onebox
         canonical_link = doc.at('//link[@rel="canonical"]/@href')
         canonical_uri = Addressable::URI.parse(canonical_link)
         if canonical_link && "#{canonical_uri.host}#{canonical_uri.path}" != "#{uri.host}#{uri.path}"
-          response = (fetch_response(canonical_uri.to_s, nil, nil, headers) rescue nil)
+          response = (fetch_response(canonical_uri.to_s, headers: headers, body_cacher: body_cacher) rescue nil)
           doc = Nokogiri::HTML(response) if response
         end
       end
@@ -45,16 +45,23 @@ module Onebox
       doc
     end
 
-    def self.fetch_response(location, limit = nil, domain = nil, headers = nil)
+    def self.fetch_response(location, redirect_limit: 5, domain: nil, headers: nil, body_cacher: nil)
+      redirect_limit = Onebox.options.redirect_limit if redirect_limit > Onebox.options.redirect_limit
 
-      limit ||= 5
-      limit = Onebox.options.redirect_limit if limit > Onebox.options.redirect_limit
-
-      raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0
+      raise Net::HTTPError.new('HTTP redirect too deep', location) if redirect_limit == 0
 
       uri = Addressable::URI.parse(location)
       uri = Addressable::URI.join(domain, uri) if !uri.host
 
+      use_body_cacher = body_cacher && body_cacher.respond_to?('fetch_cached_response_body')
+      if use_body_cacher
+        response_body = body_cacher.fetch_cached_response_body(uri.to_s)
+
+        if response_body.present?
+          return response_body
+        end
+      end
+
       result = StringIO.new
       Net::HTTP.start(uri.host, uri.port, use_ssl: uri.normalized_scheme == 'https') do |http|
         http.open_timeout = Onebox.options.connect_timeout
@@ -86,9 +93,9 @@ module Onebox
             response.error! unless [301, 302].include?(code)
             return fetch_response(
               response['location'],
-              limit - 1,
-              "#{uri.scheme}://#{uri.host}",
-              redir_header
+              redirect_limit: redirect_limit - 1,
+              domain: "#{uri.scheme}://#{uri.host}",
+              headers: redir_header
             )
           end
 
@@ -98,6 +105,10 @@ module Onebox
             raise Timeout::Error.new if (Time.now - start_time) > Onebox.options.timeout
           end
 
+          if use_body_cacher && body_cacher.cache_response_body?(uri)
+            body_cacher.cache_response_body(uri.to_s, result.string)
+          end
+
           return result.string
         end
       end

data/lib/onebox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Onebox
-  VERSION = "2.2.8"
+  VERSION = "2.2.9"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onebox
 version: !ruby/object:Gem::Version
-  version: 2.2.8
+  version: 2.2.9
 platform: ruby
 authors:
 - Joanna Zeta
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-09 00:00:00.000000000 Z
+date: 2021-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable