onebox 2.2.4 → 2.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 759d82290c29b9e1e3c97045ca14689dfc0f2f87221c1ce829b12a866abfd8a2
-  data.tar.gz: 5209c14887939549d9737409bd5dff67c2434c5a5f99596d2af93ef56aeaae6b
+  metadata.gz: f942eef9390a961ec26aa8d4301bf83eedabe577d067639ab6e0036ca20e02b9
+  data.tar.gz: b05c99bd27f025edc02475e38cc28b900bfa2bf64e1c5f9cd4be74cd5ab927a4
 SHA512:
-  metadata.gz: 719a0e4e72b2719f8ab0fd71474bfd07ef485c4dcba2c13809d5fbafa86f9b831ebe1aa7a73237c9cde28d7babc472234fa9ae701d9b18aae25621a641c883a0
-  data.tar.gz: f8c0c6c5883c48ea2811940f00c2419438e199c1e9a70761453c9aafdf176404633dc79133fc87d4014ee0e0c74a037ee8ee19b2f9b46cd85ef07a491f64cfe6
+  metadata.gz: '081ccc4c533884804d094d40b71175409077002595b10b9c3fd56d8f6c11bcacdc99502fdd78f082adb1e59d83090182550436579ec8bc2361714bb659f7a693'
+  data.tar.gz: d3dfcb8f455e7d9189415cfdf9e00f7f1d938b8ce41d8d307ff2ff4bbb6fcf09449a4a881b0bfec34f8b275556691601f2e6d070351ea2b51d5626e5a561b2fd

data/.github/workflows/ci.yml

@@ -44,6 +44,6 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Release Gem
-        uses: CvX/publish-rubygems-action@master
+        uses: discourse/publish-rubygems-action@main
         env:
           RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}

data/lib/onebox/engine/allowlisted_generic_onebox.rb

@@ -227,8 +227,10 @@ module Onebox
           d[:image] = d[:image_secure_url] || d[:image_url] || d[:thumbnail_url] || d[:image]
           d[:image] = Onebox::Helpers::get_absolute_image_url(d[:image], @url)
           d[:image] = Onebox::Helpers::normalize_url_for_output(html_entities.decode(d[:image]))
+          d[:image] = nil if Onebox::Helpers.blank?(d[:image])
 
           d[:video] = d[:video_secure_url] || d[:video_url] || d[:video]
+          d[:video] = nil if Onebox::Helpers.blank?(d[:video])
 
           d[:published_time] = d[:article_published_time] unless Onebox::Helpers.blank?(d[:article_published_time])
           if !Onebox::Helpers.blank?(d[:published_time])

data/lib/onebox/engine/amazon_onebox.rb

@@ -11,11 +11,25 @@ module Onebox
       include HTML
 
       always_https
-      matches_regexp(/^https?:\/\/(?:www\.)?(?:smile\.)?(amazon|amzn)\.(?<tld>com|ca|de|it|es|fr|co\.jp|co\.uk|cn|in|com\.br|com\.mx)\//)
+      matches_regexp(/^https?:\/\/(?:www\.)?(?:smile\.)?(amazon|amzn)\.(?<tld>com|ca|de|it|es|fr|co\.jp|co\.uk|cn|in|com\.br|com\.mx|nl|pl|sa|sg|se|com\.tr|ae)\//)
 
       def url
+        # If possible, fetch the cached HTML body immediately so we can
+        # try to grab the canonical URL from that document,
+        # rather than guess at the best URL structure to use
+        if body_cacher&.respond_to?('cache_response_body?')
+          if body_cacher.cache_response_body?(uri.to_s) && body_cacher.cached_response_body_exists?(uri.to_s)
+            @raw ||= Onebox::Helpers.fetch_html_doc(@url, http_params, body_cacher)
+          end
+        end
+
+        if @raw
+          canonical_link = @raw.at('//link[@rel="canonical"]/@href')
+          return canonical_link.to_s if canonical_link
+        end
+
         if match && match[:id]
-          return "https://www.amazon.#{tld}/gp/aw/d/#{Onebox::Helpers.uri_encode(match[:id])}"
+          return "https://www.amazon.#{tld}/dp/#{Onebox::Helpers.uri_encode(match[:id])}"
         end
 
         @url
@@ -26,16 +40,15 @@ module Onebox
       end
 
       def http_params
-        {
-          'User-Agent' =>
-          'Mozilla/5.0 (iPhone; CPU iPhone OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3'
-        }
+        if @options && @options[:user_agent]
+          { 'User-Agent' => @options[:user_agent] }
+        end
       end
 
       private
 
       def match
-        @match ||= @url.match(/(?:d|g)p\/(?:product\/|video\/detail\/)?(?<id>[^\/]+)(?:\/|$)/mi)
+        @match ||= @url.match(/(?:d|g)p\/(?:product\/|video\/detail\/)?(?<id>[A-Z0-9]+)(?:\/|\?|$)/mi)
       end
 
       def image
@@ -50,6 +63,10 @@ module Onebox
         end
 
         if (landing_image = raw.css("#landingImage")) && landing_image.any?
+          attributes = landing_image.first.attributes
+
+          return attributes["data-old-hires"].to_s if attributes["data-old-hires"]
+
           landing_image.first["src"].to_s
         end
 
@@ -100,7 +117,7 @@ module Onebox
           end
 
           result = {
-            link: link,
+            link: url,
             title: title,
             by_info: authors,
             image: og.image || image,
@@ -131,7 +148,7 @@ module Onebox
           end
 
           result = {
-            link: link,
+            link: url,
             title: title,
             by_info: authors,
             image: og.image || image,
@@ -147,7 +164,7 @@ module Onebox
         else
           title = og.title || CGI.unescapeHTML(raw.css("title").inner_text)
           result = {
-            link: link,
+            link: url,
             title: title,
             image: og.image || image,
             price: price
@@ -157,7 +174,10 @@ module Onebox
           result[:by_info] = Onebox::Helpers.clean(result[:by_info].inner_html) if result[:by_info]
 
           summary = raw.at("#productDescription")
-          result[:description] = og.description || (summary && summary.inner_text) || CGI.unescapeHTML(Onebox::Helpers.truncate(raw.css("meta[name=description]").first["content"], 250))
+
+          description = og.description || summary&.inner_text
+          description ||= raw.css("meta[name=description]").first&.[]("content")
+          result[:description] = CGI.unescapeHTML(Onebox::Helpers.truncate(description, 250)) if description
         end
 
         result[:price] = nil if result[:price].start_with?("$0") || result[:price] == 0

data/lib/onebox/engine/audio_onebox.rb

@@ -15,7 +15,7 @@ module Onebox
         escaped_url = ::Onebox::Helpers.normalize_url_for_output(@url)
 
         <<-HTML
-          <audio controls>
+          <audio controls #{@options[:disable_media_download_controls] ? 'controlslist="nodownload"' : ""}>
             <source src="#{escaped_url}">
             <a href="#{escaped_url}">#{@url}</a>
           </audio>

data/lib/onebox/engine/gfycat_onebox.rb

@@ -63,7 +63,7 @@ module Onebox
 
       def nokogiri_page
         @nokogiri_page ||= begin
-          response = Onebox::Helpers.fetch_response(url, 10) rescue nil
+          response = Onebox::Helpers.fetch_response(url, redirect_limit: 10) rescue nil
           Nokogiri::HTML(response)
         end
       end

data/lib/onebox/engine/google_docs_onebox.rb

@@ -47,7 +47,7 @@ module Onebox
       end
 
       def get_og_data
-        response = Onebox::Helpers.fetch_response(url, 10) rescue nil
+        response = Onebox::Helpers.fetch_response(url, redirect_limit: 10) rescue nil
         html = Nokogiri::HTML(response)
         og_data = {}
         html.css('meta').each do |m|

data/lib/onebox/engine/html.rb

@@ -11,7 +11,11 @@ module Onebox
       end
 
       def raw
-        @raw ||= Onebox::Helpers.fetch_html_doc(url, http_params)
+        @raw ||= Onebox::Helpers.fetch_html_doc(url, http_params, body_cacher)
+      end
+
+      def body_cacher
+        self.options&.[](:body_cacher)
       end
 
       def html?

data/lib/onebox/engine/pastebin_onebox.rb

@@ -31,7 +31,7 @@ module Onebox
 
       def lines
         return @lines if @lines
-        response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1) rescue ""
+        response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", redirect_limit: 1) rescue ""
         @lines = response.split("\n")
       end
 

data/lib/onebox/engine/twitter_status_onebox.rb

@@ -17,7 +17,7 @@ module Onebox
       private
 
       def get_twitter_data
-        response = Onebox::Helpers.fetch_response(url, nil, nil, http_params) rescue nil
+        response = Onebox::Helpers.fetch_response(url, headers: http_params) rescue nil
         html = Nokogiri::HTML(response)
         twitter_data = {}
         html.css('meta').each do |m|

data/lib/onebox/engine/video_onebox.rb

@@ -20,7 +20,7 @@ module Onebox
         escaped_url = ::Onebox::Helpers.normalize_url_for_output(@url)
         <<-HTML
           <div class="onebox video-onebox">
-            <video width='100%' height='100%' controls>
+            <video width='100%' height='100%' controls #{@options[:disable_media_download_controls] ? 'controlslist="nodownload"' : ""}>
               <source src='#{escaped_url}'>
               <a href='#{escaped_url}'>#{@url}</a>
             </video>

data/lib/onebox/helpers.rb

@@ -24,8 +24,8 @@ module Onebox
       html.gsub(/<[^>]+>/, ' ').gsub(/\n/, '')
     end
 
-    def self.fetch_html_doc(url, headers = nil)
-      response = (fetch_response(url, nil, nil, headers) rescue nil)
+    def self.fetch_html_doc(url, headers = nil, body_cacher = nil)
+      response = (fetch_response(url, headers: headers, body_cacher: body_cacher) rescue nil)
       doc = Nokogiri::HTML(response)
       uri = Addressable::URI.parse(url)
 
@@ -37,7 +37,7 @@ module Onebox
         canonical_link = doc.at('//link[@rel="canonical"]/@href')
         canonical_uri = Addressable::URI.parse(canonical_link)
         if canonical_link && "#{canonical_uri.host}#{canonical_uri.path}" != "#{uri.host}#{uri.path}"
-          response = (fetch_response(canonical_uri.to_s, nil, nil, headers) rescue nil)
+          response = (fetch_response(canonical_uri.to_s, headers: headers, body_cacher: body_cacher) rescue nil)
           doc = Nokogiri::HTML(response) if response
         end
       end
@@ -45,16 +45,23 @@ module Onebox
       doc
     end
 
-    def self.fetch_response(location, limit = nil, domain = nil, headers = nil)
+    def self.fetch_response(location, redirect_limit: 5, domain: nil, headers: nil, body_cacher: nil)
+      redirect_limit = Onebox.options.redirect_limit if redirect_limit > Onebox.options.redirect_limit
 
-      limit ||= 5
-      limit = Onebox.options.redirect_limit if limit > Onebox.options.redirect_limit
-
-      raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0
+      raise Net::HTTPError.new('HTTP redirect too deep', location) if redirect_limit == 0
 
       uri = Addressable::URI.parse(location)
       uri = Addressable::URI.join(domain, uri) if !uri.host
 
+      use_body_cacher = body_cacher && body_cacher.respond_to?('fetch_cached_response_body')
+      if use_body_cacher
+        response_body = body_cacher.fetch_cached_response_body(uri.to_s)
+
+        if response_body.present?
+          return response_body
+        end
+      end
+
       result = StringIO.new
       Net::HTTP.start(uri.host, uri.port, use_ssl: uri.normalized_scheme == 'https') do |http|
         http.open_timeout = Onebox.options.connect_timeout
@@ -86,9 +93,9 @@ module Onebox
             response.error! unless [301, 302].include?(code)
             return fetch_response(
               response['location'],
-              limit - 1,
-              "#{uri.scheme}://#{uri.host}",
-              redir_header
+              redirect_limit: redirect_limit - 1,
+              domain: "#{uri.scheme}://#{uri.host}",
+              headers: redir_header
             )
           end
 
@@ -98,6 +105,10 @@ module Onebox
             raise Timeout::Error.new if (Time.now - start_time) > Onebox.options.timeout
           end
 
+          if use_body_cacher && body_cacher.cache_response_body?(uri)
+            body_cacher.cache_response_body(uri.to_s, result.string)
+          end
+
           return result.string
         end
       end
@@ -178,6 +189,10 @@ module Onebox
       url.gsub!("'", "&apos;")
       url.gsub!('"', "&quot;")
       url.gsub!(/[^\w\-`.~:\/?#\[\]@!$&'\(\)*+,;=%\p{M}’]/, "")
+
+      parsed = Addressable::URI.parse(url)
+      return "" unless parsed.host
+
       url
     end
 

data/lib/onebox/open_graph.rb

@@ -32,7 +32,8 @@ module Onebox
       if method_name.end_with?(*integer_suffixes)
         value.to_i
       elsif method_name.end_with?(*url_suffixes)
-        ::Onebox::Helpers.normalize_url_for_output(value)
+        result = Onebox::Helpers.normalize_url_for_output(value)
+        result unless Onebox::Helpers::blank?(result)
       else
         value
       end

data/lib/onebox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Onebox
-  VERSION = "2.2.4"
+  VERSION = "2.2.10"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onebox
 version: !ruby/object:Gem::Version
-  version: 2.2.4
+  version: 2.2.10
 platform: ruby
 authors:
 - Joanna Zeta
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-24 00:00:00.000000000 Z
+date: 2021-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable