onebox 2.2.6 → 2.2.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43221e11b40d179b5a90c7c84f6c79096ba999b06ab411d159668c775de75e99
-  data.tar.gz: f23358a13d9644974b3e266f5acfd32eb13e113e085a4806369f2f858397e425
+  metadata.gz: cdbe1e67e709c8a2c8179a7118f4b51447a15a1bf1b56619fc601d3a9486123e
+  data.tar.gz: 8a72a7fc2152789c4103de330b7dcd69ec9a9c487fdc499a6068bdd0840e2e26
 SHA512:
-  metadata.gz: 290c2d0fbc5b6e2f3a4a4557e82cf6f509b8756d4c56be587af86cf95bd5c9cd58ba01e5b266b29268723941efbd542d03d89afa92bf08f0a8c87b42752e03c8
-  data.tar.gz: 5497119a20088d0b09fe69b172926ba60318cf4145a174bac66f8ef781fc7465e6cf0e9524f94156633cb605d3e9081f62697b641ce3e514c2cfc74d2a8026f3
+  metadata.gz: 3fe9a20e93b0aea3a3a7362bb5e9f026ddb63ab0c7edb64f5b7a1d9831fa76d001314ea0fdf07c92daefe95fc096e8068d28e3e6f5c7860689e5545646031084
+  data.tar.gz: 4e2e19424dab311c7a56feec82486333520f72575f95acccd1a11b8419616d2ea895e02472542d6bcdce7c5d6fbf72f887eab3941f67dd546762349bec3b66a9

data/.github/workflows/ci.yml

@@ -44,6 +44,6 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Release Gem
-        uses: CvX/publish-rubygems-action@master
+        uses: discourse/publish-rubygems-action@main
         env:
           RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}

data/lib/onebox/engine/allowlisted_generic_onebox.rb

@@ -227,8 +227,10 @@ module Onebox
           d[:image] = d[:image_secure_url] || d[:image_url] || d[:thumbnail_url] || d[:image]
           d[:image] = Onebox::Helpers::get_absolute_image_url(d[:image], @url)
           d[:image] = Onebox::Helpers::normalize_url_for_output(html_entities.decode(d[:image]))
+          d[:image] = nil if Onebox::Helpers.blank?(d[:image])
 
           d[:video] = d[:video_secure_url] || d[:video_url] || d[:video]
+          d[:video] = nil if Onebox::Helpers.blank?(d[:video])
 
           d[:published_time] = d[:article_published_time] unless Onebox::Helpers.blank?(d[:article_published_time])
           if !Onebox::Helpers.blank?(d[:published_time])

data/lib/onebox/engine/amazon_onebox.rb

@@ -11,11 +11,25 @@ module Onebox
       include HTML
 
       always_https
-      matches_regexp(/^https?:\/\/(?:www\.)?(?:smile\.)?(amazon|amzn)\.(?<tld>com|ca|de|it|es|fr|co\.jp|co\.uk|cn|in|com\.br|com\.mx)\//)
+      matches_regexp(/^https?:\/\/(?:www\.)?(?:smile\.)?(amazon|amzn)\.(?<tld>com|ca|de|it|es|fr|co\.jp|co\.uk|cn|in|com\.br|com\.mx|nl|pl|sa|sg|se|com\.tr|ae)\//)
 
       def url
+        # If possible, fetch the cached HTML body immediately so we can
+        # try to grab the canonical URL from that document,
+        # rather than guess at the best URL structure to use
+        if body_cacher&.respond_to?('cache_response_body?')
+          if body_cacher.cache_response_body?(uri.to_s) && body_cacher.cached_response_body_exists?(uri.to_s)
+            @raw ||= Onebox::Helpers.fetch_html_doc(@url, http_params, body_cacher)
+          end
+        end
+
+        if @raw
+          canonical_link = @raw.at('//link[@rel="canonical"]/@href')
+          return canonical_link.to_s if canonical_link
+        end
+
         if match && match[:id]
-          return "https://www.amazon.#{tld}/gp/aw/d/#{Onebox::Helpers.uri_encode(match[:id])}"
+          return "https://www.amazon.#{tld}/dp/#{Onebox::Helpers.uri_encode(match[:id])}"
         end
 
         @url
@@ -26,16 +40,15 @@ module Onebox
       end
 
       def http_params
-        {
-          'User-Agent' =>
-          'Mozilla/5.0 (iPhone; CPU iPhone OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3'
-        }
+        if @options && @options[:user_agent]
+          { 'User-Agent' => @options[:user_agent] }
+        end
       end
 
       private
 
       def match
-        @match ||= @url.match(/(?:d|g)p\/(?:product\/|video\/detail\/)?(?<id>[^\/]+)(?:\/|$)/mi)
+        @match ||= @url.match(/(?:d|g)p\/(?:product\/|video\/detail\/)?(?<id>[A-Z0-9]+)(?:\/|\?|$)/mi)
       end
 
       def image
@@ -50,6 +63,10 @@ module Onebox
         end
 
         if (landing_image = raw.css("#landingImage")) && landing_image.any?
+          attributes = landing_image.first.attributes
+
+          return attributes["data-old-hires"].to_s if attributes["data-old-hires"]
+
           landing_image.first["src"].to_s
         end
 
@@ -100,7 +117,7 @@ module Onebox
           end
 
           result = {
-            link: link,
+            link: url,
             title: title,
             by_info: authors,
             image: og.image || image,
@@ -131,7 +148,7 @@ module Onebox
           end
 
           result = {
-            link: link,
+            link: url,
             title: title,
             by_info: authors,
             image: og.image || image,
@@ -147,7 +164,7 @@ module Onebox
         else
           title = og.title || CGI.unescapeHTML(raw.css("title").inner_text)
           result = {
-            link: link,
+            link: url,
             title: title,
             image: og.image || image,
             price: price
@@ -157,7 +174,10 @@ module Onebox
           result[:by_info] = Onebox::Helpers.clean(result[:by_info].inner_html) if result[:by_info]
 
           summary = raw.at("#productDescription")
-          result[:description] = og.description || (summary && summary.inner_text) || CGI.unescapeHTML(Onebox::Helpers.truncate(raw.css("meta[name=description]").first["content"], 250))
+
+          description = og.description || summary&.inner_text
+          description ||= raw.css("meta[name=description]").first&.[]("content")
+          result[:description] = CGI.unescapeHTML(Onebox::Helpers.truncate(description, 250)) if description
         end
 
         result[:price] = nil if result[:price].start_with?("$0") || result[:price] == 0

data/lib/onebox/engine/gfycat_onebox.rb

@@ -63,7 +63,7 @@ module Onebox
 
       def nokogiri_page
         @nokogiri_page ||= begin
-          response = Onebox::Helpers.fetch_response(url, 10) rescue nil
+          response = Onebox::Helpers.fetch_response(url, redirect_limit: 10) rescue nil
           Nokogiri::HTML(response)
         end
       end

data/lib/onebox/engine/github_pullrequest_onebox.rb

@@ -20,6 +20,7 @@ module Onebox
         @match ||= @url.match(%r{github\.com/(?<owner>[^/]+)/(?<repository>[^/]+)/pull/(?<number>[^/]+)})
       end
 
+      GITHUB_COMMENT_REGEX = /(<!--.*?-->\r\n)/
       def data
         result = raw.clone
         result['link'] = link
@@ -31,6 +32,10 @@ module Onebox
 
         ulink = URI(link)
         result['domain'] = "#{ulink.host}/#{ulink.path.split('/')[1]}/#{ulink.path.split('/')[2]}"
+
+        body = (result['body'] || '').gsub(GITHUB_COMMENT_REGEX, '')
+        result['body'] = body.present? ? body : nil
+
         result
       end
     end

data/lib/onebox/engine/google_docs_onebox.rb

@@ -47,7 +47,7 @@ module Onebox
       end
 
       def get_og_data
-        response = Onebox::Helpers.fetch_response(url, 10) rescue nil
+        response = Onebox::Helpers.fetch_response(url, redirect_limit: 10) rescue nil
         html = Nokogiri::HTML(response)
         og_data = {}
         html.css('meta').each do |m|

data/lib/onebox/engine/html.rb

@@ -11,7 +11,11 @@ module Onebox
       end
 
       def raw
-        @raw ||= Onebox::Helpers.fetch_html_doc(url, http_params)
+        @raw ||= Onebox::Helpers.fetch_html_doc(url, http_params, body_cacher)
+      end
+
+      def body_cacher
+        self.options&.[](:body_cacher)
       end
 
       def html?

data/lib/onebox/engine/pastebin_onebox.rb

@@ -31,7 +31,7 @@ module Onebox
 
       def lines
         return @lines if @lines
-        response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1) rescue ""
+        response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", redirect_limit: 1) rescue ""
         @lines = response.split("\n")
       end
 

data/lib/onebox/engine/twitter_status_onebox.rb

@@ -17,7 +17,7 @@ module Onebox
       private
 
       def get_twitter_data
-        response = Onebox::Helpers.fetch_response(url, nil, nil, http_params) rescue nil
+        response = Onebox::Helpers.fetch_response(url, headers: http_params) rescue nil
         html = Nokogiri::HTML(response)
         twitter_data = {}
         html.css('meta').each do |m|

data/lib/onebox/engine/youtube_onebox.rb

@@ -13,11 +13,36 @@ module Onebox
       WIDTH  ||= 480
       HEIGHT ||= 360
 
-      def placeholder_html
-        og = get_opengraph.data
+      def parse_embed_response
+        return unless video_id
+        return @parse_embed_response if defined?(@parse_embed_response)
+
+        embed_url = "https://www.youtube.com/embed/#{video_id}"
+        @embed_doc ||= Onebox::Helpers.fetch_html_doc(embed_url)
+
+        begin
+          script_tag = @embed_doc.xpath('//script').find { |tag| tag.to_s.include?('ytcfg.set') }.to_s
+          match = script_tag.to_s.match(/ytcfg\.set\((?<json>.*)\)/)
+
+          yt_json = ::JSON.parse(match[:json])
+          renderer = ::JSON.parse(yt_json['PLAYER_VARS']['embedded_player_response'])['embedPreview']['thumbnailPreviewRenderer']
+
+          title = renderer['title']['runs'].first['text']
+
+          image = "https://img.youtube.com/vi/#{video_id}/hqdefault.jpg"
+        rescue
+          return
+        end
+
+        @parse_embed_response = { image: image, title: title }
+      end
 
+      def placeholder_html
         if video_id || list_id
-          "<img src='#{og[:image]}' width='#{WIDTH}' height='#{HEIGHT}' title='#{og[:title]}'>"
+          result = parse_embed_response
+          result ||= get_opengraph.data
+
+          "<img src='#{result[:image]}' width='#{WIDTH}' height='#{HEIGHT}' title='#{result[:title]}'>"
         else
           to_html
         end
@@ -52,7 +77,10 @@ module Onebox
       end
 
       def video_title
-        @video_title ||= get_opengraph.data[:title]
+        @video_title ||= begin
+          result = parse_embed_response || get_opengraph.data
+          result[:title]
+        end
       end
 
       private

data/lib/onebox/helpers.rb

@@ -7,7 +7,7 @@ module Onebox
 
     class DownloadTooLarge < StandardError; end
 
-    IGNORE_CANONICAL_DOMAINS ||= ['www.instagram.com']
+    IGNORE_CANONICAL_DOMAINS ||= ['www.instagram.com', 'youtube.com']
 
     def self.symbolize_keys(hash)
       return {} if hash.nil?
@@ -24,8 +24,8 @@ module Onebox
       html.gsub(/<[^>]+>/, ' ').gsub(/\n/, '')
     end
 
-    def self.fetch_html_doc(url, headers = nil)
-      response = (fetch_response(url, nil, nil, headers) rescue nil)
+    def self.fetch_html_doc(url, headers = nil, body_cacher = nil)
+      response = (fetch_response(url, headers: headers, body_cacher: body_cacher) rescue nil)
       doc = Nokogiri::HTML(response)
       uri = Addressable::URI.parse(url)
 
@@ -37,7 +37,7 @@ module Onebox
         canonical_link = doc.at('//link[@rel="canonical"]/@href')
         canonical_uri = Addressable::URI.parse(canonical_link)
         if canonical_link && "#{canonical_uri.host}#{canonical_uri.path}" != "#{uri.host}#{uri.path}"
-          response = (fetch_response(canonical_uri.to_s, nil, nil, headers) rescue nil)
+          response = (fetch_response(canonical_uri.to_s, headers: headers, body_cacher: body_cacher) rescue nil)
           doc = Nokogiri::HTML(response) if response
         end
       end
@@ -45,16 +45,23 @@ module Onebox
       doc
     end
 
-    def self.fetch_response(location, limit = nil, domain = nil, headers = nil)
+    def self.fetch_response(location, redirect_limit: 5, domain: nil, headers: nil, body_cacher: nil)
+      redirect_limit = Onebox.options.redirect_limit if redirect_limit > Onebox.options.redirect_limit
 
-      limit ||= 5
-      limit = Onebox.options.redirect_limit if limit > Onebox.options.redirect_limit
-
-      raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0
+      raise Net::HTTPError.new('HTTP redirect too deep', location) if redirect_limit == 0
 
       uri = Addressable::URI.parse(location)
       uri = Addressable::URI.join(domain, uri) if !uri.host
 
+      use_body_cacher = body_cacher && body_cacher.respond_to?('fetch_cached_response_body')
+      if use_body_cacher
+        response_body = body_cacher.fetch_cached_response_body(uri.to_s)
+
+        if response_body.present?
+          return response_body
+        end
+      end
+
       result = StringIO.new
       Net::HTTP.start(uri.host, uri.port, use_ssl: uri.normalized_scheme == 'https') do |http|
         http.open_timeout = Onebox.options.connect_timeout
@@ -86,9 +93,9 @@ module Onebox
             response.error! unless [301, 302].include?(code)
             return fetch_response(
               response['location'],
-              limit - 1,
-              "#{uri.scheme}://#{uri.host}",
-              redir_header
+              redirect_limit: redirect_limit - 1,
+              domain: "#{uri.scheme}://#{uri.host}",
+              headers: redir_header
             )
           end
 
@@ -98,6 +105,10 @@ module Onebox
             raise Timeout::Error.new if (Time.now - start_time) > Onebox.options.timeout
           end
 
+          if use_body_cacher && body_cacher.cache_response_body?(uri)
+            body_cacher.cache_response_body(uri.to_s, result.string)
+          end
+
           return result.string
         end
       end
@@ -178,6 +189,10 @@ module Onebox
       url.gsub!("'", "&apos;")
       url.gsub!('"', "&quot;")
       url.gsub!(/[^\w\-`.~:\/?#\[\]@!$&'\(\)*+,;=%\p{M}’]/, "")
+
+      parsed = Addressable::URI.parse(url)
+      return "" unless parsed.host
+
       url
     end
 

data/lib/onebox/open_graph.rb

@@ -32,7 +32,8 @@ module Onebox
       if method_name.end_with?(*integer_suffixes)
         value.to_i
       elsif method_name.end_with?(*url_suffixes)
-        ::Onebox::Helpers.normalize_url_for_output(value)
+        result = Onebox::Helpers.normalize_url_for_output(value)
+        result unless Onebox::Helpers::blank?(result)
       else
         value
       end

data/lib/onebox/sanitize_config.rb

@@ -6,7 +6,7 @@ class Sanitize
     HTTP_PROTOCOLS ||= ['http', 'https', :relative].freeze
 
     ONEBOX ||= freeze_config merge(RELAXED,
-      elements: RELAXED[:elements] + %w[audio embed iframe source video svg path],
+      elements: RELAXED[:elements] + %w[audio details embed iframe source video svg path],
 
       attributes: {
         'a' => RELAXED[:attributes]['a'] + %w(target),

data/lib/onebox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Onebox
-  VERSION = "2.2.6"
+  VERSION = "2.2.12"
 end

data/templates/githubpullrequest.mustache

@@ -4,9 +4,22 @@
   </div>
 
   <div class="github-info-container">
-    <h4>
-      <a href="{{html_url}}" target="_blank" rel="noopener">{{title}}</a>
-    </h4>
+    {{^body}}
+      <h4>
+        <a href="{{html_url}}" target="_blank" rel="noopener">{{title}}</a>
+      </h4>
+    {{/body}}
+
+    {{#body}}
+      <details class="onebox-details">
+        <summary class="onebox-details-summary">
+          <h4>
+            <a href="{{html_url}}" target="_blank" rel="noopener">{{title}}</a>
+          </h4>
+        </summary>
+        <p class="onebox-details-body">{{body}}</p>
+      </details>
+    {{/body}}
 
     <div class="branches">
       <code>{{base.label}}</code> ← <code>{{head.label}}</code>

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: onebox
 version: !ruby/object:Gem::Version
-  version: 2.2.6
+  version: 2.2.12
 platform: ruby
 authors:
 - Joanna Zeta
 - Vyki Englert
 - Robin Ward
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-25 00:00:00.000000000 Z
+date: 2021-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -405,7 +405,7 @@ homepage: https://github.com/discourse/onebox
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -421,7 +421,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A gem for generating embeddable HTML previews from URLs.
 test_files: []