onebox 1.9.28.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b2bb566d2096445a37f5328daab6bd9750c8fa75ce8740b7f2f858f56c9aa795
-  data.tar.gz: 1f4ce2ee9e5255a0889c3019649a9a34802a3b4d7c7d4494ab3425d98566d471
+  metadata.gz: 2a487ded9a3811b625e50e77656011a13996919443570a91d4a409625388d9f1
+  data.tar.gz: e3638c662cd35ecc7bb043c56a5d049931c17b250232dada802a75c37fe19dec
 SHA512:
-  metadata.gz: 5e48b7a4787c335b9eefba80baa1b49d19e394b3a05ab1c43ed30fc176e5d0021a301ddbedcbf8d5a1e02649fadb41d8cd62d8668b3d0d62e030effffc667453
-  data.tar.gz: 8fbb2770d5ffa95e3efed6fb24c5f15468b659392e40bb5532abe7e7d4a677678dd2705e354efb8f33e4622c7608dac21bd6f446145a342c6683ec47d6cff178
+  metadata.gz: 0bbc8c191618dbc0639eebc0c7c427bb461f565d14c85ea933837ed5765916a1d75ca948684de8a754bd8f4181363ad8da134f292966d841377024c952ddef51
+  data.tar.gz: c31848cdf2735a4a3f5fa9873e90d5f89b13829c01381c7e78d5298f587fbba1035505bd33dba8b17cce4ad3a75d41a8ceea54bc4ba20f8fa7f63236fce41e51

data/README.md

@@ -46,6 +46,14 @@ preview = Onebox.preview(url)
 "#{preview}" == preview.to_s #=> true
 ```
 
+### Twitch Onebox
+
+To be able to embed Twitch video and clips, pass `hostname` in the options to `Onebox.preview`
+
+```ruby
+preview = Onebox.preview(url, hostname: 'www.example.com')
+```
+
 Ruby Support
 ------------
 
@@ -63,12 +71,11 @@ out the project. You can then try out URLs.
 The server doesn't reload code changes automatically (PRs accepted!) so
 make sure to hit CTRL-C and restart the server to try a code change out.
 
-
 Adding Support for a new URL
 ----------------------------
 
   1. Check if the site supports [oEmbed](http://oembed.com/) or [Open Graph](https://developers.facebook.com/docs/opengraph/).
-     If it does, you can probably get away with just whitelisting the URL in `Onebox::Engine::WhitelistedGenericOnebox` (see: [Whitelisted Generic Onebox caveats](#user-content-whitelisted-generic-onebox-caveats)).
+     If it does, you can probably get away with just allowing the URL in `Onebox::Engine::AllowlistedGenericOnebox` (see: [Allowlisted Generic Onebox caveats](#user-content-allowlisted-generic-onebox-caveats)).
      If the site does not support open standards, you can create a new engine.
 
   2. Create new onebox engine
@@ -156,16 +163,23 @@ Adding Support for a new URL
      require_relative "engine/name_onebox"
      ```
 
-
-Whitelisted Generic Onebox caveats
+Allowlisted Generic Onebox caveats
 ----------------------------------
 
-The Whitelisted Generic Onebox has some caveats for its use, beyond simply whitelisting the domain.
+The Allowlisted Generic Onebox has some caveats for its use, beyond simply allowlisting the domain.
 
-  1. The domain must be whitelisted
+  1. The domain must be allowlisted
   2. The URL you're oneboxing cannot be a root url (e.g. `http://example.com` won't work, but `http://example.com/page` will)
   3. If the oneboxed URL responds with oEmbed and has a `rich` type: the `html` content must contain an `<iframe>`. Responses without an iframe will not be oneboxed.
 
+Ignoring Canonical URLs
+-----------------------
+
+Onebox prefers to use canonical URLs instead of the raw inputted URL when searching for Open Graph metadata. If your site's canonical URL does not have opengraph metadata, use the `og:ignore_canonical` property to have Onebox ignore the canonical URL.
+
+```html
+<meta property="og:ignore_canonical" content="true" />
+```
 
 Installing
 ----------

data/lib/onebox/engine.rb

@@ -141,7 +141,7 @@ require_relative "engine/wikimedia_onebox"
 require_relative "engine/wikipedia_onebox"
 require_relative "engine/youtube_onebox"
 require_relative "engine/youku_onebox"
-require_relative "engine/whitelisted_generic_onebox"
+require_relative "engine/allowlisted_generic_onebox"
 require_relative "engine/pubmed_onebox"
 require_relative "engine/soundcloud_onebox"
 require_relative "engine/imgur_onebox"
@@ -168,7 +168,6 @@ require_relative "engine/twitch_clips_onebox"
 require_relative "engine/twitch_stream_onebox"
 require_relative "engine/twitch_video_onebox"
 require_relative "engine/trello_onebox"
-require_relative "engine/wechat_mp_onebox"
 require_relative "engine/cloudapp_onebox"
 require_relative "engine/wistia_onebox"
 require_relative "engine/simplecast_onebox"

data/lib/onebox/engine/{whitelisted_generic_onebox.rb → allowlisted_generic_onebox.rb}

@@ -4,20 +4,20 @@ require 'htmlentities'
 
 module Onebox
   module Engine
-    class WhitelistedGenericOnebox
+    class AllowlistedGenericOnebox
       include Engine
       include StandardEmbed
       include LayoutSupport
 
-      def self.whitelist=(list)
-        @whitelist = list
+      def self.allowed_domains=(list)
+        @allowed_domains = list
       end
 
-      def self.whitelist
-        @whitelist ||= default_whitelist.dup
+      def self.allowed_domains
+        @allowed_domains ||= default_allowed_domains.dup
       end
 
-      def self.default_whitelist
+      def self.default_allowed_domains
         %w(
           23hq.com
           500px.com
@@ -176,13 +176,13 @@ module Onebox
         !!(uri.path =~ /\d{4}\/\d{2}\//)
       end
 
-      def self.twitter_label_whitelist
+      def self.allowed_twitter_labels
         ['brand', 'price', 'usd', 'cad', 'reading time', 'likes']
       end
 
       def self.===(other)
         other.kind_of?(URI) ?
-          host_matches(other, whitelist) || probable_wordpress(other) || probable_discourse(other) :
+          host_matches(other, allowed_domains) || probable_wordpress(other) || probable_discourse(other) :
           super
       end
 
@@ -233,11 +233,11 @@ module Onebox
           end
 
           # Twitter labels
-          if !Onebox::Helpers.blank?(d[:label1]) && !Onebox::Helpers.blank?(d[:data1]) && !!WhitelistedGenericOnebox.twitter_label_whitelist.find { |l| d[:label1] =~ /#{l}/i }
+          if !Onebox::Helpers.blank?(d[:label1]) && !Onebox::Helpers.blank?(d[:data1]) && !!AllowlistedGenericOnebox.allowed_twitter_labels.find { |l| d[:label1] =~ /#{l}/i }
             d[:label_1] = Onebox::Helpers.truncate(d[:label1])
             d[:data_1]  = Onebox::Helpers.truncate(d[:data1])
           end
-          if !Onebox::Helpers.blank?(d[:label2]) && !Onebox::Helpers.blank?(d[:data2]) && !!WhitelistedGenericOnebox.twitter_label_whitelist.find { |l| d[:label2] =~ /#{l}/i }
+          if !Onebox::Helpers.blank?(d[:label2]) && !Onebox::Helpers.blank?(d[:data2]) && !!AllowlistedGenericOnebox.allowed_twitter_labels.find { |l| d[:label2] =~ /#{l}/i }
             unless Onebox::Helpers.blank?(d[:label_1])
               d[:label_2] = Onebox::Helpers.truncate(d[:label2])
               d[:data_2]  = Onebox::Helpers.truncate(d[:data2])
@@ -261,7 +261,7 @@ module Onebox
       def rewrite_https(html)
         return unless html
         uri = URI(@url)
-        if WhitelistedGenericOnebox.host_matches(uri, WhitelistedGenericOnebox.rewrites)
+        if AllowlistedGenericOnebox.host_matches(uri, AllowlistedGenericOnebox.rewrites)
           html = html.gsub("http://", "https://")
         end
         html
@@ -309,7 +309,7 @@ module Onebox
         data[:height] &&
         (
           data[:html]["iframe"] ||
-          WhitelistedGenericOnebox.html_providers.include?(data[:provider_name])
+          AllowlistedGenericOnebox.html_providers.include?(data[:provider_name])
         )
       end
 

data/lib/onebox/engine/audio_onebox.rb

@@ -8,7 +8,7 @@ module Onebox
       matches_regexp(/^(https?:)?\/\/.*\.(mp3|ogg|opus|wav|m4a)(\?.*)?$/i)
 
       def always_https?
-        WhitelistedGenericOnebox.host_matches(uri, WhitelistedGenericOnebox.https_hosts)
+        AllowlistedGenericOnebox.host_matches(uri, AllowlistedGenericOnebox.https_hosts)
       end
 
       def to_html

data/lib/onebox/engine/facebook_media_onebox.rb

@@ -22,7 +22,7 @@ module Onebox
             </iframe>
           HTML
         else
-          html = Onebox::Engine::WhitelistedGenericOnebox.new(@url, @timeout).to_html
+          html = Onebox::Engine::AllowlistedGenericOnebox.new(@url, @timeout).to_html
           return if Onebox::Helpers.blank?(html)
           html
         end

data/lib/onebox/engine/gfycat_onebox.rb

@@ -10,7 +10,7 @@ module Onebox
       always_https
 
       def self.priority
-        # This engine should have priority over WhitelistedGenericOnebox.
+        # This engine should have priority over AllowlistedGenericOnebox.
         1
       end
 

data/lib/onebox/engine/google_photos_onebox.rb

@@ -20,11 +20,25 @@ module Onebox
       private
 
       def video_html(og)
+        escaped_url = ::Onebox::Helpers.normalize_url_for_output(url)
+
         <<-HTML
-            <video width='#{og.video_width}' height='#{og.video_height}' #{og.title_attr} poster="#{og.get_secure_image}" controls loop>
-              <source src='#{og.video_secure_url}' type='video/mp4'>
-            </video>
-          HTML
+          <aside class="onebox google-photos">
+            <header class="source">
+              <img src="#{raw[:favicon]}" class="site-icon" width="16" height="16">
+              <a href="#{escaped_url}" target="_blank" rel="nofollow ugc noopener">#{raw[:site_name]}</a>
+            </header>
+            <article class="onebox-body">
+              <h3><a href="#{escaped_url}" target="_blank" rel="nofollow ugc noopener">#{og.title}</a></h3>
+              <div class="aspect-image-full-size">
+                <a href="#{escaped_url}" target="_blank" rel="nofollow ugc noopener">
+                  <img src="#{og.get_secure_image}" class="scale-image"/>
+                  <span class="instagram-video-icon"></span>
+                </a>
+              </div>
+            </article>
+          </aside>
+        HTML
       end
 
       def album_html(og)

data/lib/onebox/engine/image_onebox.rb

@@ -8,7 +8,7 @@ module Onebox
       matches_regexp(/^(https?:)?\/\/.+\.(png|jpg|jpeg|gif|bmp|tif|tiff)(\?.*)?$/i)
 
       def always_https?
-        WhitelistedGenericOnebox.host_matches(uri, WhitelistedGenericOnebox.https_hosts)
+        AllowlistedGenericOnebox.host_matches(uri, AllowlistedGenericOnebox.https_hosts)
       end
 
       def to_html

data/lib/onebox/engine/instagram_onebox.rb

@@ -17,27 +17,12 @@ module Onebox
       def data
         oembed = get_oembed
         permalink = clean_url.gsub("/#{oembed.author_name}/", "/")
-        description = oembed.title
-        type = if description =~ /^Photos by/
-          "album"
-        elsif description =~ /^Video by/
-          "video"
-        else
-          "photo"
-        end
-        title = if type == "album"
-          "[Album] @#{oembed.author_name}"
-        else
-          "@#{oembed.author_name}"
-        end
 
-        result = { link: permalink,
-                   title: title,
-                   image: "#{permalink}/media/?size=l",
-                   description: Onebox::Helpers.truncate(description, 250)
-                  }
-        result[:video_link] = permalink if type == "video"
-        result
+        { link: permalink,
+          title: "@#{oembed.author_name}",
+          image: "#{permalink}/media/?size=l",
+          description: Onebox::Helpers.truncate(oembed.title, 250)
+        }
       end
 
       protected

data/lib/onebox/engine/reddit_media_onebox.rb

@@ -45,7 +45,7 @@ module Onebox
             </aside>
           HTML
         else
-          html = Onebox::Engine::WhitelistedGenericOnebox.new(@url, @timeout).to_html
+          html = Onebox::Engine::AllowlistedGenericOnebox.new(@url, @timeout).to_html
           return if Onebox::Helpers.blank?(html)
           html
         end

data/lib/onebox/engine/standard_embed.rb

@@ -32,7 +32,7 @@ module Onebox
       add_oembed_provider(/nytimes\.com\//, 'https://www.nytimes.com/svc/oembed/json/')
 
       def always_https?
-        WhitelistedGenericOnebox.host_matches(uri, WhitelistedGenericOnebox.https_hosts) || super
+        AllowlistedGenericOnebox.host_matches(uri, AllowlistedGenericOnebox.https_hosts) || super
       end
 
       def raw

data/lib/onebox/engine/twitter_status_onebox.rb

@@ -10,10 +10,14 @@ module Onebox
       matches_regexp(/^https?:\/\/(mobile\.|www\.)?twitter\.com\/.+?\/status(es)?\/\d+(\/(video|photo)\/\d?+)?+(\/?\?.*)?\/?$/)
       always_https
 
+      def http_params
+        { 'User-Agent' => 'DiscourseBot/1.0' }
+      end
+
       private
 
       def get_twitter_data
-        response = Onebox::Helpers.fetch_response(url) rescue nil
+        response = Onebox::Helpers.fetch_response(url, nil, nil, http_params) rescue nil
         html = Nokogiri::HTML(response)
         twitter_data = {}
         html.css('meta').each do |m|

data/lib/onebox/engine/video_onebox.rb

@@ -8,7 +8,7 @@ module Onebox
       matches_regexp(/^(https?:)?\/\/.*\.(mov|mp4|webm|ogv)(\?.*)?$/i)
 
       def always_https?
-        WhitelistedGenericOnebox.host_matches(uri, WhitelistedGenericOnebox.https_hosts)
+        AllowlistedGenericOnebox.host_matches(uri, AllowlistedGenericOnebox.https_hosts)
       end
 
       def to_html

data/lib/onebox/engine/youtube_onebox.rb

@@ -45,7 +45,7 @@ module Onebox
           HTML
         else
           # for channel pages
-          html = Onebox::Engine::WhitelistedGenericOnebox.new(@url, @timeout).to_html
+          html = Onebox::Engine::AllowlistedGenericOnebox.new(@url, @timeout).to_html
           return if Onebox::Helpers.blank?(html)
           html.gsub!(/['"]\/\//, "https://")
           html

data/lib/onebox/helpers.rb

@@ -51,17 +51,14 @@ module Onebox
 
       raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0
 
-      uri = URI(location)
-      uri = URI("#{domain}#{location}") if !uri.host
+      uri = Addressable::URI.parse(location)
+      uri = Addressable::URI.join(domain, uri) if !uri.host
 
       result = StringIO.new
-      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http|
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.normalized_scheme == 'https') do |http|
         http.open_timeout = Onebox.options.connect_timeout
         http.read_timeout = Onebox.options.timeout
-        if uri.is_a?(URI::HTTPS)
-          http.use_ssl = true
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        end
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE  # Work around path building bugs
 
         headers ||= {}
 
@@ -76,10 +73,12 @@ module Onebox
         http.request(request) do |response|
 
           if cookie = response.get_fields('set-cookie')
-            header = { 'Cookie' => cookie.join }
+            # HACK: If this breaks again in the future, use HTTP::CookieJar from gem 'http-cookie'
+            # See test: it "does not send cookies to the wrong domain"
+            redir_header = { 'Cookie' => cookie.join('; ') }
           end
 
-          header = nil unless header.is_a? Hash
+          redir_header = nil unless redir_header.is_a? Hash
 
           code = response.code.to_i
           unless code === 200
@@ -88,7 +87,7 @@ module Onebox
               response['location'],
               limit - 1,
               "#{uri.scheme}://#{uri.host}",
-              header
+              redir_header
             )
           end
 

data/lib/onebox/matcher.rb

@@ -16,7 +16,10 @@ module Onebox
       uri = URI(@url)
       return unless uri.port.nil? || Onebox.options.allowed_ports.include?(uri.port)
       return unless uri.scheme.nil? || Onebox.options.allowed_schemes.include?(uri.scheme)
-      ordered_engines.find { |engine| engine === uri }
+      ordered_engines
+        .select { |engine| engine === uri }
+        .sort_by { |engine| engine.to_s }
+        .last
     rescue URI::InvalidURIError
       nil
     end

data/lib/onebox/mixins/twitch_onebox.rb

@@ -24,7 +24,9 @@ module Onebox
         end
 
         def to_html
-          "<iframe src=\"//#{base_url}#{query_params}&autoplay=false\" width=\"620\" height=\"378\" frameborder=\"0\" style=\"overflow: hidden;\" scrolling=\"no\" allowfullscreen=\"allowfullscreen\"></iframe>"
+          <<~HTML
+          <iframe src="//#{base_url}#{query_params}&parent=#{options[:hostname]}&autoplay=false" width="620" height="378" frameborder="0" style="overflow: hidden;" scrolling="no" allowfullscreen="allowfullscreen"></iframe>
+          HTML
         end
       end
     end

data/lib/onebox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Onebox
-  VERSION = "1.9.28.2"
+  VERSION = "2.0.0"
 end

data/templates/instagram.mustache

@@ -2,9 +2,9 @@
 
 {{#image}}
   <div class="instagram-images">
-    {{#video_link}} <a href="{{{video_link}}}" target="_blank" rel="noopener"> {{/video_link}}
+    <a href="{{{link}}}" target="_blank" rel="noopener">
       <img class="instagram-image" src="{{{image}}}"/>
-    {{#video_link}} <span class="instagram-video-icon"></span></a> {{/video_link}}
+    </a>
   </div>
 {{/image}}
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onebox
 version: !ruby/object:Gem::Version
-  version: 1.9.28.2
+  version: 2.0.0
 platform: ruby
 authors:
 - Joanna Zeta
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-26 00:00:00.000000000 Z
+date: 2020-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -300,6 +300,7 @@ files:
 - Rakefile
 - lib/onebox.rb
 - lib/onebox/engine.rb
+- lib/onebox/engine/allowlisted_generic_onebox.rb
 - lib/onebox/engine/amazon_onebox.rb
 - lib/onebox/engine/asciinema_onebox.rb
 - lib/onebox/engine/audio_onebox.rb
@@ -353,8 +354,6 @@ files:
 - lib/onebox/engine/typeform_onebox.rb
 - lib/onebox/engine/video_onebox.rb
 - lib/onebox/engine/vimeo_onebox.rb
-- lib/onebox/engine/wechat_mp_onebox.rb
-- lib/onebox/engine/whitelisted_generic_onebox.rb
 - lib/onebox/engine/wikimedia_onebox.rb
 - lib/onebox/engine/wikipedia_onebox.rb
 - lib/onebox/engine/wistia_onebox.rb
@@ -380,6 +379,7 @@ files:
 - lib/onebox/web_helpers.rb
 - onebox.gemspec
 - templates/_layout.mustache
+- templates/allowlistedgeneric.mustache
 - templates/amazon.mustache
 - templates/githubblob.mustache
 - templates/githubcommit.mustache
@@ -396,8 +396,6 @@ files:
 - templates/pubmed.mustache
 - templates/stackexchange.mustache
 - templates/twitterstatus.mustache
-- templates/wechatmp.mustache
-- templates/whitelistedgeneric.mustache
 - templates/wikimedia.mustache
 - templates/wikipedia.mustache
 - templates/xkcd.mustache

data/lib/onebox/engine/wechat_mp_onebox.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-module Onebox
-  module Engine
-    class WechatMpOnebox
-      include Engine
-      include LayoutSupport
-      include HTML
-
-      always_https
-      matches_regexp(/^https?:\/\/mp\.weixin\.qq\.com\/s.*$/)
-
-      def tld
-        @tld || @@matcher.match(@url)["tld"]
-      end
-
-      def http_params
-        {
-          'User-Agent' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3',
-          'Accept-Encoding' => 'plain'
-        }
-      end
-
-      private
-
-      def extract_script_value(var_name)
-        if (script_elem = raw.css("script").select { |script| script.inner_text.include? "var #{var_name} = " }) && script_elem.any?
-          e = Nokogiri::HTML(script_elem[0].inner_text.match(/var\s+#{Regexp.quote(var_name)}\s+=\s+"(.*?)";/)[1])
-          CGI::unescapeHTML(e.text.scan(/(?:\\x([a-f0-9]{2}))|(.)/i).map { |x| x[0] ? [x[0].to_i(16)].pack('U') : x[1] }.join)
-        end
-      end
-
-      # TODO need to handle hotlink protection from wechat
-      def image
-        if banner_image = extract_script_value("msg_cdn_url")
-          return banner_image
-        end
-
-        if (main_image = raw.css("img").select { |img| not img['class'] }) && main_image.any?
-          attributes = main_image.first.attributes
-
-          return attributes["data-src"].to_s if attributes["data-src"]
-        end
-      end
-
-      def data
-        title = CGI.unescapeHTML(raw.css("title").inner_text)
-        by_info = CGI.unescapeHTML(raw.css("span.rich_media_meta_text.rich_media_meta_nickname").inner_text)
-
-        result = {
-          link: extract_script_value("msg_link") || link,
-          title: title,
-          image: image,
-          description: extract_script_value("msg_desc"),
-          by_info: by_info
-        }
-
-        result
-      end
-    end
-  end
-end

data/templates/wechatmp.mustache

@@ -1,4 +0,0 @@
-<h3><a href='{{link}}' target='_blank' rel='noopener'>{{title}}</a></h3>
-{{#by_info}}<b>{{by_info}}</b>{{/by_info}}
-<p>{{description}}</p>
-