onebox 2.2.2 → 2.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 817b81480e38b1a133342d89f83991cd0e94217581ddb74c6275c6631c7285bf
-  data.tar.gz: 545e95966e368c2b36a9792dd99934ec609635e19d95bb02ef1dd1fbcd51c5f6
+  metadata.gz: 541807b758c73492e2868271a63b8f05ef4d94aad316be86fb5c42c3c3f0e2ce
+  data.tar.gz: 10f460d3d6a1b09a6087a38ad0692c9f652e027beb65b22845536480055b4a48
 SHA512:
-  metadata.gz: fc0ea7daff3babaa77cebf9e5e00161877a86106d15b1a2d4e1b1e56f1e4ba9ebfd6e3666a991d420e4ea71180dc8fcaf2bafc0ac659934399706462c1e45d69
-  data.tar.gz: f509adf0a3ad1d557700a690bfb9b922ec9c181b68cf6ab61134394ee70c8ee4487d64faa863262a28cd795ed14e39bc774521572551c719faa39a1acc208a3e
+  metadata.gz: d13068e9dbb437388ee689581bc64bb08bf4c5ae898755680c3c978a00fc4a4ee13004f0f2a3071f40d226d15e84f799a949d0d07572c6537ddd92950416d56b
+  data.tar.gz: 1efacb58ff5d24daff63cdd855a32ae80826f3585eee0305ba032269ee4431f1541848582bd5587aacffdcd6cb9c5bb35c650706c6028ed5e115e17daf362621

data/.github/workflows/ci.yml

@@ -15,24 +15,19 @@ jobs:
     strategy:
       matrix:
         ruby:
-          - 2.4
           - 2.5
           - 2.6
+          - 2.7
+          - 3.0
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
 
       - name: Setup ruby
-        uses: actions/setup-ruby@v1
+        uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-          architecture: 'x64'
-
-      - name: Setup bundler
-        run: gem install bundler
-
-      - name: Setup gems
-        run: bundle install
+          bundler-cache: true
 
       - name: Rubocop
         run: bundle exec rubocop
@@ -49,6 +44,6 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Release Gem
-        uses: CvX/publish-rubygems-action@master
+        uses: discourse/publish-rubygems-action@main
         env:
           RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}

data/lib/onebox/engine/allowlisted_generic_onebox.rb

@@ -227,8 +227,10 @@ module Onebox
           d[:image] = d[:image_secure_url] || d[:image_url] || d[:thumbnail_url] || d[:image]
           d[:image] = Onebox::Helpers::get_absolute_image_url(d[:image], @url)
           d[:image] = Onebox::Helpers::normalize_url_for_output(html_entities.decode(d[:image]))
+          d[:image] = nil if Onebox::Helpers.blank?(d[:image])
 
           d[:video] = d[:video_secure_url] || d[:video_url] || d[:video]
+          d[:video] = nil if Onebox::Helpers.blank?(d[:video])
 
           d[:published_time] = d[:article_published_time] unless Onebox::Helpers.blank?(d[:article_published_time])
           if !Onebox::Helpers.blank?(d[:published_time])

data/lib/onebox/engine/amazon_onebox.rb

@@ -11,11 +11,22 @@ module Onebox
       include HTML
 
       always_https
-      matches_regexp(/^https?:\/\/(?:www\.)?(?:smile\.)?(amazon|amzn)\.(?<tld>com|ca|de|it|es|fr|co\.jp|co\.uk|cn|in|com\.br|com\.mx)\//)
+      matches_regexp(/^https?:\/\/(?:www\.)?(?:smile\.)?(amazon|amzn)\.(?<tld>com|ca|de|it|es|fr|co\.jp|co\.uk|cn|in|com\.br|com\.mx|nl|pl|sa|sg|se|com\.tr|ae)\//)
 
       def url
+        # Have we cached the HTML body of the requested URL?
+        # If so, try to grab the canonical URL from that document,
+        # rather than guess at the best URL structure to use
+        if @body_cacher && @body_cacher.respond_to?('cache_response_body?')
+          if @body_cacher.cached_response_body_exists?(uri.to_s)
+            @raw ||= Onebox::Helpers.fetch_html_doc(@url, http_params, @body_cacher)
+            canonical_link = @raw.at('//link[@rel="canonical"]/@href')
+            return canonical_link.to_s if canonical_link
+          end
+        end
+
         if match && match[:id]
-          return "https://www.amazon.#{tld}/gp/aw/d/#{Onebox::Helpers.uri_encode(match[:id])}"
+          return "https://www.amazon.#{tld}/dp/#{Onebox::Helpers.uri_encode(match[:id])}"
         end
 
         @url
@@ -26,10 +37,9 @@ module Onebox
       end
 
       def http_params
-        {
-          'User-Agent' =>
-          'Mozilla/5.0 (iPhone; CPU iPhone OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3'
-        }
+        if @options && @options[:user_agent]
+          { 'User-Agent' => @options[:user_agent] }
+        end
       end
 
       private

data/lib/onebox/engine/audio_onebox.rb

@@ -15,7 +15,7 @@ module Onebox
         escaped_url = ::Onebox::Helpers.normalize_url_for_output(@url)
 
         <<-HTML
-          <audio controls>
+          <audio controls #{@options[:disable_media_download_controls] ? 'controlslist="nodownload"' : ""}>
             <source src="#{escaped_url}">
             <a href="#{escaped_url}">#{@url}</a>
           </audio>

data/lib/onebox/engine/gfycat_onebox.rb

@@ -63,7 +63,7 @@ module Onebox
 
       def nokogiri_page
         @nokogiri_page ||= begin
-          response = Onebox::Helpers.fetch_response(url, 10) rescue nil
+          response = Onebox::Helpers.fetch_response(url, redirect_limit: 10) rescue nil
           Nokogiri::HTML(response)
         end
       end

data/lib/onebox/engine/github_issue_onebox.rb

@@ -22,8 +22,7 @@ module Onebox
       end
 
       def data
-
-        @raw ||= ::MultiJson.load(open(url, "Accept" => "application/vnd.github.v3.text+json", read_timeout: timeout)) #custom Accept header so we can get body as text.
+        @raw ||= ::MultiJson.load(URI.open(url, "Accept" => "application/vnd.github.v3.text+json", read_timeout: timeout)) #custom Accept header so we can get body as text.
         body_text = @raw["body_text"]
 
         content_words = body_text.gsub("\n\n", "\n").gsub("\n", "<br>").split(" ") #one pass of removing double newline, then we change \n to <br> and later on we revert it back to \n this is a workaround to avoid losing newlines after we join it back.

data/lib/onebox/engine/google_docs_onebox.rb

@@ -47,7 +47,7 @@ module Onebox
       end
 
       def get_og_data
-        response = Onebox::Helpers.fetch_response(url, 10) rescue nil
+        response = Onebox::Helpers.fetch_response(url, redirect_limit: 10) rescue nil
         html = Nokogiri::HTML(response)
         og_data = {}
         html.css('meta').each do |m|

data/lib/onebox/engine/html.rb

@@ -11,7 +11,8 @@ module Onebox
       end
 
       def raw
-        @raw ||= Onebox::Helpers.fetch_html_doc(url, http_params)
+        body_cacher = self.options[:body_cacher] if self.options
+        @raw ||= Onebox::Helpers.fetch_html_doc(url, http_params, body_cacher)
       end
 
       def html?

data/lib/onebox/engine/json.rb

@@ -6,7 +6,7 @@ module Onebox
       private
 
       def raw
-        @raw ||= ::MultiJson.load(open(url, read_timeout: timeout))
+        @raw ||= ::MultiJson.load(URI.open(url, read_timeout: timeout))
       end
     end
   end

data/lib/onebox/engine/pastebin_onebox.rb

@@ -31,7 +31,7 @@ module Onebox
 
       def lines
         return @lines if @lines
-        response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1) rescue ""
+        response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", redirect_limit: 1) rescue ""
         @lines = response.split("\n")
       end
 

data/lib/onebox/engine/pubmed_onebox.rb

@@ -11,7 +11,7 @@ module Onebox
       private
 
       def get_xml
-        doc = Nokogiri::XML(open(URI.join(@url, "?report=xml&format=text")))
+        doc = Nokogiri::XML(URI.open(URI.join(@url, "?report=xml&format=text")))
         pre = doc.xpath("//pre")
         Nokogiri::XML("<root>" + pre.text + "</root>")
       end

data/lib/onebox/engine/twitter_status_onebox.rb

@@ -17,7 +17,7 @@ module Onebox
       private
 
       def get_twitter_data
-        response = Onebox::Helpers.fetch_response(url, nil, nil, http_params) rescue nil
+        response = Onebox::Helpers.fetch_response(url, headers: http_params) rescue nil
         html = Nokogiri::HTML(response)
         twitter_data = {}
         html.css('meta').each do |m|

data/lib/onebox/engine/video_onebox.rb

@@ -20,7 +20,7 @@ module Onebox
         escaped_url = ::Onebox::Helpers.normalize_url_for_output(@url)
         <<-HTML
           <div class="onebox video-onebox">
-            <video width='100%' height='100%' controls>
+            <video width='100%' height='100%' controls #{@options[:disable_media_download_controls] ? 'controlslist="nodownload"' : ""}>
               <source src='#{escaped_url}'>
               <a href='#{escaped_url}'>#{@url}</a>
             </video>

data/lib/onebox/helpers.rb

@@ -24,8 +24,8 @@ module Onebox
       html.gsub(/<[^>]+>/, ' ').gsub(/\n/, '')
     end
 
-    def self.fetch_html_doc(url, headers = nil)
-      response = (fetch_response(url, nil, nil, headers) rescue nil)
+    def self.fetch_html_doc(url, headers = nil, body_cacher = nil)
+      response = (fetch_response(url, headers: headers, body_cacher: body_cacher) rescue nil)
       doc = Nokogiri::HTML(response)
       uri = Addressable::URI.parse(url)
 
@@ -37,7 +37,7 @@ module Onebox
         canonical_link = doc.at('//link[@rel="canonical"]/@href')
         canonical_uri = Addressable::URI.parse(canonical_link)
         if canonical_link && "#{canonical_uri.host}#{canonical_uri.path}" != "#{uri.host}#{uri.path}"
-          response = (fetch_response(canonical_uri.to_s, nil, nil, headers) rescue nil)
+          response = (fetch_response(canonical_uri.to_s, headers: headers, body_cacher: body_cacher) rescue nil)
           doc = Nokogiri::HTML(response) if response
         end
       end
@@ -45,16 +45,23 @@ module Onebox
       doc
     end
 
-    def self.fetch_response(location, limit = nil, domain = nil, headers = nil)
+    def self.fetch_response(location, redirect_limit: 5, domain: nil, headers: nil, body_cacher: nil)
+      redirect_limit = Onebox.options.redirect_limit if redirect_limit > Onebox.options.redirect_limit
 
-      limit ||= 5
-      limit = Onebox.options.redirect_limit if limit > Onebox.options.redirect_limit
-
-      raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0
+      raise Net::HTTPError.new('HTTP redirect too deep', location) if redirect_limit == 0
 
       uri = Addressable::URI.parse(location)
       uri = Addressable::URI.join(domain, uri) if !uri.host
 
+      use_body_cacher = body_cacher && body_cacher.respond_to?('fetch_cached_response_body')
+      if use_body_cacher
+        response_body = body_cacher.fetch_cached_response_body(uri.to_s)
+
+        if response_body.present?
+          return response_body
+        end
+      end
+
       result = StringIO.new
       Net::HTTP.start(uri.host, uri.port, use_ssl: uri.normalized_scheme == 'https') do |http|
         http.open_timeout = Onebox.options.connect_timeout
@@ -86,9 +93,9 @@ module Onebox
             response.error! unless [301, 302].include?(code)
             return fetch_response(
               response['location'],
-              limit - 1,
-              "#{uri.scheme}://#{uri.host}",
-              redir_header
+              redirect_limit: redirect_limit - 1,
+              domain: "#{uri.scheme}://#{uri.host}",
+              headers: redir_header
             )
           end
 
@@ -98,6 +105,10 @@ module Onebox
             raise Timeout::Error.new if (Time.now - start_time) > Onebox.options.timeout
           end
 
+          if use_body_cacher && body_cacher.cache_response_body?(uri)
+            body_cacher.cache_response_body(uri.to_s, result.string)
+          end
+
           return result.string
         end
       end
@@ -178,6 +189,10 @@ module Onebox
       url.gsub!("'", "&apos;")
       url.gsub!('"', "&quot;")
       url.gsub!(/[^\w\-`.~:\/?#\[\]@!$&'\(\)*+,;=%\p{M}’]/, "")
+
+      parsed = Addressable::URI.parse(url)
+      return "" unless parsed.host
+
       url
     end
 

data/lib/onebox/mixins/git_blob_onebox.rb

@@ -167,7 +167,7 @@ module Onebox
               @raw = "https://render.githubusercontent.com/view/solid?url=" + self.raw_template(m)
 
             else
-              contents = open(self.raw_template(m), read_timeout: timeout).read
+              contents = URI.open(self.raw_template(m), read_timeout: timeout).read
 
               contents_lines = contents.lines           #get contents lines
               contents_lines_size = contents_lines.size #get number of lines

data/lib/onebox/open_graph.rb

@@ -32,7 +32,8 @@ module Onebox
       if method_name.end_with?(*integer_suffixes)
         value.to_i
       elsif method_name.end_with?(*url_suffixes)
-        ::Onebox::Helpers.normalize_url_for_output(value)
+        result = Onebox::Helpers.normalize_url_for_output(value)
+        result unless Onebox::Helpers::blank?(result)
       else
         value
       end

data/lib/onebox/sanitize_config.rb

@@ -10,7 +10,7 @@ class Sanitize
 
       attributes: {
         'a' => RELAXED[:attributes]['a'] + %w(target),
-        'audio' => %w[controls],
+        'audio' => %w[controls controlslist],
         'embed' => %w[height src type width],
         'iframe' => %w[allowfullscreen frameborder height scrolling src width data-original-href data-unsanitized-src],
         'source' => %w[src type],

data/lib/onebox/status_check.rb

@@ -35,7 +35,7 @@ module Onebox
     private
 
     def check
-      res = open(@url, read_timeout: (@options.timeout || Onebox.options.timeout))
+      res = URI.open(@url, read_timeout: (@options.timeout || Onebox.options.timeout))
       @status = res.status.first.to_i
     rescue OpenURI::HTTPError => e
       @status = e.io.status.first.to_i

data/lib/onebox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Onebox
-  VERSION = "2.2.2"
+  VERSION = "2.2.9"
 end

data/onebox.gemspec

@@ -41,5 +41,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'haml', '~> 5.1'
   spec.add_development_dependency 'listen', '~> 2.10.0'
 
-  spec.required_ruby_version = '>=2.4.0'
+  spec.required_ruby_version = '>=2.5.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onebox
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: 2.2.9
 platform: ruby
 authors:
 - Joanna Zeta
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-06 00:00:00.000000000 Z
+date: 2021-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -413,7 +413,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="