onebox 2.2.8 → 2.2.13

data/lib/onebox/engine/trello_onebox.rb

@@ -16,7 +16,7 @@ module Onebox
         height = match[:type] == 'b' ? 400 : 200
 
         <<-HTML
-          <iframe src=\"#{link}\" width=\"100%\" height=\"#{height}\" frameborder=\"0\" style=\"border:0\"></iframe>
+          <iframe src="#{link}" width="100%" height="#{height}" frameborder="0" style="border:0"></iframe>
         HTML
       end
 
@@ -26,7 +26,7 @@ module Onebox
 
       private
       def match
-        return @match if @match
+        return @match if defined?(@match)
 
         @match = @url.match(%{trello\.com/(?<type>[^/]+)/(?<key>[^/]+)/?\W*})
 

data/lib/onebox/engine/twitch_clips_onebox.rb

@@ -3,12 +3,11 @@
 require_relative '../mixins/twitch_onebox'
 
 class Onebox::Engine::TwitchClipsOnebox
-
   def self.twitch_regexp
     /^https?:\/\/clips\.twitch\.tv\/([a-zA-Z0-9_]+\/?[^#\?\/]+)/
   end
-  include Onebox::Mixins::TwitchOnebox
 
+  include Onebox::Mixins::TwitchOnebox
   requires_iframe_origins "https://clips.twitch.tv"
 
   def query_params
@@ -18,5 +17,4 @@ class Onebox::Engine::TwitchClipsOnebox
   def base_url
     "clips.twitch.tv/embed?"
   end
-
 end

data/lib/onebox/engine/twitch_stream_onebox.rb

@@ -3,14 +3,13 @@
 require_relative '../mixins/twitch_onebox'
 
 class Onebox::Engine::TwitchStreamOnebox
-
   def self.twitch_regexp
     /^https?:\/\/(?:www\.|go\.)?twitch\.tv\/(?!directory)([a-zA-Z0-9_]{4,25})$/
   end
+
   include Onebox::Mixins::TwitchOnebox
 
   def query_params
     "channel=#{twitch_id}"
   end
-
 end

data/lib/onebox/engine/twitch_video_onebox.rb

@@ -3,7 +3,6 @@
 require_relative '../mixins/twitch_onebox'
 
 class Onebox::Engine::TwitchVideoOnebox
-
   def self.twitch_regexp
     /^https?:\/\/(?:www\.)?twitch\.tv\/videos\/([0-9]+)/
   end
@@ -13,5 +12,4 @@ class Onebox::Engine::TwitchVideoOnebox
   def query_params
     "video=v#{twitch_id}"
   end
-
 end

data/lib/onebox/engine/twitter_status_onebox.rb

@@ -17,7 +17,7 @@ module Onebox
       private
 
       def get_twitter_data
-        response = Onebox::Helpers.fetch_response(url, nil, nil, http_params) rescue nil
+        response = Onebox::Helpers.fetch_response(url, headers: http_params) rescue nil
         html = Nokogiri::HTML(response)
         twitter_data = {}
         html.css('meta').each do |m|

data/lib/onebox/engine/typeform_onebox.rb

@@ -13,12 +13,13 @@ module Onebox
         typeform_src = build_typeform_src
 
         <<~HTML
-          <iframe src="#{typeform_src}"
-                  width="100%"
-                  height="600px"
-                  scrolling="no"
-                  frameborder="0">
-          </iframe>
+          <iframe
+            src="#{typeform_src}"
+            width="100%"
+            height="600px"
+            scrolling="no"
+            frameborder="0"
+          ></iframe>
         HTML
       end
 

data/lib/onebox/engine/vimeo_onebox.rb

@@ -25,14 +25,16 @@ module Onebox
         end
         video_src = "https://player.vimeo.com/video/#{video_id}"
         video_src = video_src.gsub('autoplay=1', '').chomp("?")
+
         <<-HTML
-          <iframe width="#{WIDTH}"
-                  height="#{HEIGHT}"
-                  src="#{video_src}"
-                  data-original-href="#{link}"
-                  frameborder="0"
-                  allowfullscreen>
-          </iframe>
+          <iframe
+            width="#{WIDTH}"
+            height="#{HEIGHT}"
+            src="#{video_src}"
+            data-original-href="#{link}"
+            frameborder="0"
+            allowfullscreen
+          ></iframe>
         HTML
       end
 

data/lib/onebox/engine/wikimedia_onebox.rb

@@ -7,7 +7,7 @@ module Onebox
       include LayoutSupport
       include JSON
 
-      matches_regexp /^https?:\/\/commons\.wikimedia\.org\/wiki\/(File:.+)/
+      matches_regexp(/^https?:\/\/commons\.wikimedia\.org\/wiki\/(File:.+)/)
       always_https
 
       def self.priority
@@ -38,7 +38,6 @@ module Onebox
           thumbnail: first_page['imageinfo'].first['thumburl']
         }
       end
-
     end
   end
 end

data/lib/onebox/engine/wikipedia_onebox.rb

@@ -16,10 +16,10 @@ module Onebox
         paras = []
         text = ""
 
-        # Detect section Hash in the url and retrive the related paragraphs.  if no hash provided the first few paragraphs will be used
+        # Detect section Hash in the url and retrive the related paragraphs. if no hash provided the first few paragraphs will be used
         # Author Lidlanca
         # Date 9/8/2014
-        if (m_url_hash = @url.match(/#([^\/?]+)/)) #extract url hash
+        if (m_url_hash = @url.match(/#([^\/?]+)/)) # extract url hash
           m_url_hash_name = m_url_hash[1]
         end
 
@@ -27,17 +27,17 @@ module Onebox
           section_header_title = raw.xpath("//span[@id='#{m_url_hash_name}']")
 
           if section_header_title.empty?
-            paras = raw.search("p") #default get all the paras
+            paras = raw.search("p") # default get all the paras
           else
             section_title_text = section_header_title.inner_text
-            section_header = section_header_title[0].parent #parent element of the section span element should be an <h3> node
+            section_header = section_header_title[0].parent # parent element of the section span element should be an <h3> node
             cur_element = section_header
 
-            # p|text|div covers the general case. We assume presence of atleast 1 P node. if section has no P node we may end up with a P node from the next section.
+            # p|text|div covers the general case. We assume presence of at least 1 P node. if section has no P node we may end up with a P node from the next section.
             # div tag is commonly used as an assets wraper in an article section. often as the first element holding an image.
             # ul support will imporve the output generated for a section with a list as the main content (for example: an Author Bibliography, A musician Discography, etc)
             first_p_found = nil
-            while (((next_sibling = cur_element.next_sibling).name =~ /p|text|div|ul/) || first_p_found.nil?) do  #from section header get the next sibling until it is a breaker tag
+            while (((next_sibling = cur_element.next_sibling).name =~ /p|text|div|ul/) || first_p_found.nil?) do  # from section header get the next sibling until it is a breaker tag
               cur_element = next_sibling
               if (cur_element.name == "p" || cur_element.name == "ul") #we treat a list as we detect a p to avoid showing
                 first_p_found = true
@@ -46,7 +46,7 @@ module Onebox
             end
           end
         else # no hash found in url
-          paras = raw.search("p") #default get all the paras
+          paras = raw.search("p") # default get all the paras
         end
 
         unless paras.empty?
@@ -55,7 +55,7 @@ module Onebox
             break if cnt >= paras.size
             text += " " unless cnt == 0
 
-            if paras[cnt].name == "ul" #Handle UL tag. Generate a textual ordered list (1.item | 2.item | 3.item). Unfourtently no newline allowed in output
+            if paras[cnt].name == "ul" # Handle UL tag. Generate a textual ordered list (1.item | 2.item | 3.item). Unfortunately no newline allowed in output
               li_index = 1
               list_items = []
               paras[cnt].children.css("li").each { |li| list_items.push "#{li_index}." + li.inner_text ; li_index += 1 }
@@ -69,13 +69,17 @@ module Onebox
             cnt += 1
           end
         end
+
         text = "#{text[0..Onebox::LayoutSupport.max_text]}..." if text.length > Onebox::LayoutSupport.max_text
+
         result = {
           link: link,
           title: raw.css("html body h1").inner_text + (section_title_text ? " | " + section_title_text : ""),  #if a section sub title exists add it to the main article title
           description: text
         }
+
         img = raw.css(".image img")
+
         if img && img.size > 0
           img.each do |i|
             src = i["src"]

data/lib/onebox/engine/youku_onebox.rb

@@ -21,12 +21,13 @@ module Onebox
 
       def to_html
         <<~HTML
-          <iframe src="https://player.youku.com/embed/#{video_id}"
-                  width="640"
-                  height="430"
-                  frameborder='0'
-                  allowfullscreen>
-          </iframe>
+          <iframe
+            src="https://player.youku.com/embed/#{video_id}"
+            width="640"
+            height="430"
+            frameborder='0'
+            allowfullscreen
+          ></iframe>
         HTML
       end
 
@@ -35,7 +36,6 @@ module Onebox
       def uri
         @_uri ||= URI(@url)
       end
-
     end
   end
 end

data/lib/onebox/engine/youtube_onebox.rb

@@ -13,11 +13,36 @@ module Onebox
       WIDTH  ||= 480
       HEIGHT ||= 360
 
-      def placeholder_html
-        og = get_opengraph.data
+      def parse_embed_response
+        return unless video_id
+        return @parse_embed_response if defined?(@parse_embed_response)
+
+        embed_url = "https://www.youtube.com/embed/#{video_id}"
+        @embed_doc ||= Onebox::Helpers.fetch_html_doc(embed_url)
+
+        begin
+          script_tag = @embed_doc.xpath('//script').find { |tag| tag.to_s.include?('ytcfg.set') }.to_s
+          match = script_tag.to_s.match(/ytcfg\.set\((?<json>.*)\)/)
+
+          yt_json = ::JSON.parse(match[:json])
+          renderer = ::JSON.parse(yt_json['PLAYER_VARS']['embedded_player_response'])['embedPreview']['thumbnailPreviewRenderer']
+
+          title = renderer['title']['runs'].first['text']
+
+          image = "https://img.youtube.com/vi/#{video_id}/hqdefault.jpg"
+        rescue
+          return
+        end
 
+        @parse_embed_response = { image: image, title: title }
+      end
+
+      def placeholder_html
         if video_id || list_id
-          "<img src='#{og[:image]}' width='#{WIDTH}' height='#{HEIGHT}' title='#{og[:title]}'>"
+          result = parse_embed_response
+          result ||= get_opengraph.data
+
+          "<img src='#{result[:image]}' width='#{WIDTH}' height='#{HEIGHT}' title='#{result[:title]}'>"
         else
           to_html
         end
@@ -26,21 +51,23 @@ module Onebox
       def to_html
         if video_id
           <<-HTML
-            <iframe width="#{WIDTH}"
-                    height="#{HEIGHT}"
-                    src="https://www.youtube.com/embed/#{video_id}?#{embed_params}"
-                    frameborder="0"
-                    allowfullscreen>
-            </iframe>
+            <iframe
+              src="https://www.youtube.com/embed/#{video_id}?#{embed_params}"
+              width="#{WIDTH}"
+              height="#{HEIGHT}"
+              frameborder="0"
+              allowfullscreen
+            ></iframe>
           HTML
         elsif list_id
           <<-HTML
-            <iframe width="#{WIDTH}"
-                    height="#{HEIGHT}"
-                    src="https://www.youtube.com/embed/videoseries?list=#{list_id}&wmode=transparent&rel=0&autohide=1&showinfo=1&enablejsapi=1"
-                    frameborder="0"
-                    allowfullscreen>
-            </iframe>
+            <iframe
+              src="https://www.youtube.com/embed/videoseries?list=#{list_id}&wmode=transparent&rel=0&autohide=1&showinfo=1&enablejsapi=1"
+              width="#{WIDTH}"
+              height="#{HEIGHT}"
+              frameborder="0"
+              allowfullscreen
+            ></iframe>
           HTML
         else
           # for channel pages
@@ -52,7 +79,10 @@ module Onebox
       end
 
       def video_title
-        @video_title ||= get_opengraph.data[:title]
+        @video_title ||= begin
+          result = parse_embed_response || get_opengraph.data
+          result[:title]
+        end
       end
 
       private
@@ -138,7 +168,6 @@ module Onebox
       rescue
         {}
       end
-
     end
   end
 end

data/lib/onebox/file_type_finder.rb

@@ -2,7 +2,6 @@
 
 module Onebox
   module FileTypeFinder
-
     # In general, most of file extension names would be recognized
     # by Highlights.js.  However, some need to be checked in other
     # ways, either because they just aren't included, because they

data/lib/onebox/helpers.rb

@@ -7,7 +7,7 @@ module Onebox
 
     class DownloadTooLarge < StandardError; end
 
-    IGNORE_CANONICAL_DOMAINS ||= ['www.instagram.com']
+    IGNORE_CANONICAL_DOMAINS ||= ['www.instagram.com', 'youtube.com']
 
     def self.symbolize_keys(hash)
       return {} if hash.nil?
@@ -24,8 +24,8 @@ module Onebox
       html.gsub(/<[^>]+>/, ' ').gsub(/\n/, '')
     end
 
-    def self.fetch_html_doc(url, headers = nil)
-      response = (fetch_response(url, nil, nil, headers) rescue nil)
+    def self.fetch_html_doc(url, headers = nil, body_cacher = nil)
+      response = (fetch_response(url, headers: headers, body_cacher: body_cacher) rescue nil)
       doc = Nokogiri::HTML(response)
       uri = Addressable::URI.parse(url)
 
@@ -37,7 +37,7 @@ module Onebox
         canonical_link = doc.at('//link[@rel="canonical"]/@href')
         canonical_uri = Addressable::URI.parse(canonical_link)
         if canonical_link && "#{canonical_uri.host}#{canonical_uri.path}" != "#{uri.host}#{uri.path}"
-          response = (fetch_response(canonical_uri.to_s, nil, nil, headers) rescue nil)
+          response = (fetch_response(canonical_uri.to_s, headers: headers, body_cacher: body_cacher) rescue nil)
           doc = Nokogiri::HTML(response) if response
         end
       end
@@ -45,16 +45,23 @@ module Onebox
       doc
     end
 
-    def self.fetch_response(location, limit = nil, domain = nil, headers = nil)
+    def self.fetch_response(location, redirect_limit: 5, domain: nil, headers: nil, body_cacher: nil)
+      redirect_limit = Onebox.options.redirect_limit if redirect_limit > Onebox.options.redirect_limit
 
-      limit ||= 5
-      limit = Onebox.options.redirect_limit if limit > Onebox.options.redirect_limit
-
-      raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0
+      raise Net::HTTPError.new('HTTP redirect too deep', location) if redirect_limit == 0
 
       uri = Addressable::URI.parse(location)
       uri = Addressable::URI.join(domain, uri) if !uri.host
 
+      use_body_cacher = body_cacher && body_cacher.respond_to?('fetch_cached_response_body')
+      if use_body_cacher
+        response_body = body_cacher.fetch_cached_response_body(uri.to_s)
+
+        if response_body.present?
+          return response_body
+        end
+      end
+
       result = StringIO.new
       Net::HTTP.start(uri.host, uri.port, use_ssl: uri.normalized_scheme == 'https') do |http|
         http.open_timeout = Onebox.options.connect_timeout
@@ -86,9 +93,9 @@ module Onebox
             response.error! unless [301, 302].include?(code)
             return fetch_response(
               response['location'],
-              limit - 1,
-              "#{uri.scheme}://#{uri.host}",
-              redir_header
+              redirect_limit: redirect_limit - 1,
+              domain: "#{uri.scheme}://#{uri.host}",
+              headers: redir_header
             )
           end
 
@@ -98,6 +105,10 @@ module Onebox
             raise Timeout::Error.new if (Time.now - start_time) > Onebox.options.timeout
           end
 
+          if use_body_cacher && body_cacher.cache_response_body?(uri)
+            body_cacher.cache_response_body(uri.to_s, result.string)
+          end
+
           return result.string
         end
       end
@@ -116,10 +127,10 @@ module Onebox
 
         http.request_head([uri.path, uri.query].join("?")) do |response|
           code = response.code.to_i
-          unless code === 200 || Onebox::Helpers.blank?(response.header['content-length'])
+          unless code === 200 || Onebox::Helpers.blank?(response.content_length)
             return nil
           end
-          return response.header['content-length']
+          return response.content_length
         end
       end
     end

data/lib/onebox/mixins/git_blob_onebox.rb

@@ -33,6 +33,10 @@ module Onebox
 
           self.options = DEFAULTS
 
+          @selected_lines_array = nil
+          @selected_one_liner = 0
+          @model_file = nil
+
           # Define constant after merging options set in Onebox.options
           # We can define constant automatically.
           options.each_pair do |constant_name, value|
@@ -47,8 +51,6 @@ module Onebox
         end
 
         private
-        @selected_lines_array = nil
-        @selected_one_liner = 0
 
         def calc_range(m, contents_lines_size)
           truncated = false
@@ -150,7 +152,7 @@ module Onebox
         end
 
         def raw
-          return @raw if @raw
+          return @raw if defined?(@raw)
 
           m = @url.match(self.raw_regexp)