onebox 1.7.5 → 1.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b37cb209211e4d83b000bf4143ceb893ee46c1d
-  data.tar.gz: ff64989d2bb635abd2178b8a67e6fc47aab94aaf
+  metadata.gz: 9ee1f0a234b8d376cceb7543476abc508345e061
+  data.tar.gz: 9374c5a627bab394767c26f8efa157000b36f3b6
 SHA512:
-  metadata.gz: 81943e75131d21db1bccf904a7549277c390b5c0a000b203a988292a55454c18100fb01c39054bec04ac5ffd51cc88cbe22fc6ab58fe2e43b50949eaeb5ad886
-  data.tar.gz: 6125b92caca985a46e607e94b00df093840be649353191914e11b81071ea874c394860a1dc159e1862d2e6554332860839a63cea13da91d55f5bc6830b24fa0c
+  metadata.gz: afe7b6be89693cbca5ecf84dc97d00e904ec19ba3120f0e97252d3bfad42dd1de97a5caecc783ac2246804bfc20f1c5665b3e97cb72939ddd1ffdc9d72e05963
+  data.tar.gz: 76a0d2d867c9de7f0fec490af7e33b3f2294b9bfeab2ec14047d5866fd502dd9a501a7acc541ff5b271d936836e7ddea47ffb1c9e28888c1003cbee84ab49e6a

data/lib/onebox.rb

@@ -15,6 +15,7 @@ module Onebox
     cache: Moneta.new(:Memory, expires: true, serializer: :json),
     connect_timeout: 5,
     timeout: 10,
+    max_download_kb: (10 * 1024),  # 10MB
     load_paths: [File.join(Gem::Specification.find_by_name("onebox").gem_dir, "templates")]
   }
 

data/lib/onebox/engine/google_docs_onebox.rb

@@ -45,8 +45,8 @@ module Onebox
       end
 
       def get_og_data
-        response = Onebox::Helpers.fetch_response(url, 10)
-        html = Nokogiri::HTML(response.body)
+        response = Onebox::Helpers.fetch_response(url, 10) rescue nil
+        html = Nokogiri::HTML(response)
         og_data = {}
         html.css('meta').each do |m|
           if m.attribute('property') && m.attribute('property').to_s.match(/^og:/i)

data/lib/onebox/engine/imgur_onebox.rb

@@ -47,7 +47,8 @@ module Onebox
         end
 
         def is_album?
-          oembed_data = Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response("http://api.imgur.com/oembed.json?url=#{url}").body))
+          response = Onebox::Helpers.fetch_response("http://api.imgur.com/oembed.json?url=#{url}") rescue "{}"
+          oembed_data = Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
           imgur_data_id = Nokogiri::HTML(oembed_data[:html]).xpath("//blockquote").attr("data-id")
           imgur_data_id.to_s[/a\//]
         end

data/lib/onebox/engine/pastebin_onebox.rb

@@ -29,8 +29,8 @@ module Onebox
 
       def lines
         return @lines if @lines
-        response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1)
-        @lines = response.body.split("\n")
+        response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1) rescue ""
+        @lines = response.split("\n")
       end
 
       def paste_key

data/lib/onebox/engine/soundcloud_onebox.rb

@@ -23,7 +23,8 @@ module Onebox
           @oembed_data ||= begin
             oembed_url = "https://soundcloud.com/oembed.json?url=#{url}"
             oembed_url << "&maxheight=166" unless url["/sets/"]
-            Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(oembed_url).body))
+            response = Onebox::Helpers.fetch_response(oembed_url) rescue "{}"
+            Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
           rescue
             {}
           end

data/lib/onebox/engine/standard_embed.rb

@@ -50,7 +50,7 @@ module Onebox
       protected
 
         def html_doc
-          @html_doc ||= Nokogiri::HTML(Onebox::Helpers.fetch_response(url).body) rescue nil
+          @html_doc ||= Nokogiri::HTML((Onebox::Helpers.fetch_response(url) rescue nil))
         end
 
         def get_oembed
@@ -77,7 +77,8 @@ module Onebox
 
           return {} if Onebox::Helpers.blank?(oembed_url)
 
-          oe = Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(oembed_url).body))
+          json_response = Onebox::Helpers.fetch_response(oembed_url) rescue "{}"
+          oe = Onebox::Helpers.symbolize_keys(::MultiJson.load(json_response))
 
           # never use oembed from WordPress 4.4 (it's broken)
           oe.delete(:html) if oe[:html] && oe[:html]["wp-embedded-content"]

data/lib/onebox/engine/twitter_status_onebox.rb

@@ -5,14 +5,14 @@ module Onebox
       include LayoutSupport
       include HTML
 
-      matches_regexp Regexp.new("^https?://(?:www\\.)?(?:(?:\\w)+\\.)?(twitter)\\.com(?:/)?(?:.)*/status(es)?/")
+      matches_regexp /^https?:\/\/(mobile\.|www\.)?twitter\.com\/.+?\/status(es)?\/\d+$/
       always_https
 
       private
 
       def get_twitter_data
-        response = Onebox::Helpers.fetch_response(url)
-        html = Nokogiri::HTML(response.body)
+        response = Onebox::Helpers.fetch_response(url) rescue nil
+        html = Nokogiri::HTML(response)
         twitter_data = {}
         html.css('meta').each do |m|
           if m.attribute('property') && m.attribute('property').to_s.match(/^og:/i)

data/lib/onebox/engine/youtube_onebox.rb

@@ -84,7 +84,8 @@ module Onebox
         def list_thumbnail_url
           @list_thumbnail_url ||= begin
             url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/playlist?list=#{list_id}"
-            data = Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(url).body))
+            response = Onebox::Helpers.fetch_response(url) rescue "{}"
+            data = Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
             data[:thumbnail_url]
           rescue
             nil
@@ -93,12 +94,14 @@ module Onebox
 
         def video_oembed_data
           url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/watch?v=#{video_id}"
-          Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(url).body))
+          response = Onebox::Helpers.fetch_response(url) rescue "{}"
+          Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
         end
 
         def list_oembed_data
           url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/playlist?list=#{list_id}"
-          Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(url).body))
+          response = Onebox::Helpers.fetch_response(url) rescue "{}"
+          Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
         end
 
         def embed_params

data/lib/onebox/helpers.rb

@@ -1,5 +1,8 @@
 module Onebox
   module Helpers
+
+    class DownloadTooLarge < Exception; end;
+
     def self.symbolize_keys(hash)
       return {} if hash.nil?
 
@@ -15,33 +18,54 @@ module Onebox
       html.gsub(/<[^>]+>/, ' ').gsub(/\n/, '')
     end
 
-    def self.fetch_response(location, limit = 5, domain = nil, headers = nil)
+    def self.fetch_response(location, limit=5, domain=nil, headers=nil)
       raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0
 
       uri = URI(location)
       uri = URI("#{domain}#{location}") if !uri.host
 
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.open_timeout = Onebox.options.connect_timeout
-      http.read_timeout = Onebox.options.timeout
-      if uri.is_a?(URI::HTTPS)
-        http.use_ssl = true
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      end
+      result = StringIO.new
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http|
+        http.open_timeout = Onebox.options.connect_timeout
+        http.read_timeout = Onebox.options.timeout
+        if uri.is_a?(URI::HTTPS)
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
 
-      response = http.request_get(uri.request_uri,headers)
+        request = Net::HTTP::Get.new(uri.request_uri, headers)
+        start_time = Time.now
 
-      if cookie = response.get_fields('set-cookie')
-        header = { 'cookie' => cookie.join }
-      end
+        puts Onebox.options.max_download_kb
+        size_bytes = Onebox.options.max_download_kb * 1024
+        puts "size_byes: #{size_bytes}"
+        http.request(request) do |response|
+
+          if cookie = response.get_fields('set-cookie')
+            header = { 'cookie' => cookie.join }
+          end
+
+          header = nil unless header.is_a? Hash
+
+          code = response.code.to_i
+          unless code === 200
+            response.error! unless [301, 302].include?(code)
+            return fetch_response(
+              response['location'],
+              limit - 1,
+              "#{uri.scheme}://#{uri.host}",
+              header
+            )
+          end
 
-      header = nil unless header.is_a? Hash
+          response.read_body do |chunk|
+            result.write(chunk)
+            raise DownloadTooLarge.new if result.size > size_bytes
+            raise Timeout::Error.new if (Time.now - start_time) > Onebox.options.timeout
+          end
 
-      case response
-        when Net::HTTPSuccess     then response
-        when Net::HTTPRedirection then fetch_response(response['location'], limit - 1, "#{uri.scheme}://#{uri.host}",header)
-        else
-          response.error!
+          return result.string
+        end
       end
     end
 
@@ -71,7 +95,7 @@ module Onebox
       # expect properly encoded url, remove any unsafe chars
       url.gsub!("'", "&apos;")
       url.gsub!('"', "&quot;")
-      url.gsub!(/[^\w\-`._~:\/?#\[\]@!$&'\(\)*+,;=]/, "")
+      url.gsub!(/[^\w\-`.~:\/?#\[\]@!$&'\(\)*+,;=]/, "")
       url
     end
 

data/lib/onebox/version.rb

@@ -1,3 +1,3 @@
 module Onebox
-  VERSION = "1.7.5"
+  VERSION = "1.7.6"
 end

data/spec/lib/onebox/engine/whitelisted_generic_onebox_spec.rb

@@ -84,19 +84,23 @@ describe Onebox::Engine::WhitelistedGenericOnebox do
   end
 
   describe 'to_html' do
-
     before do
       described_class.whitelist = %w(dailymail.co.uk discourse.org)
       original_link = "http://www.dailymail.co.uk/pages/live/articles/news/news.html?in_article_id=479146&in_page_id=1770"
       redirect_link = 'http://www.dailymail.co.uk/news/article-479146/Brutality-justice-The-truth-tarred-feathered-drug-dealer.html'
-      FakeWeb.register_uri(:get, original_link, status: ["301", "Moved Permanently"], location: '/news/article-479146/Brutality-justice-The-truth-tarred-feathered-drug-dealer.html')
+      FakeWeb.register_uri(
+        :get,
+        original_link,
+        status: ["301", "Moved Permanently"],
+        location: redirect_link
+      )
       fake(redirect_link, response('dailymail'))
       onebox = described_class.new(original_link)
       @html = onebox.to_html
     end
     let(:html) { @html }
 
-    it "includes summary" do
+    it "follows redirects and includes the summary" do
       expect(html).to include("It was the most chilling image of the week")
     end
   end

data/spec/lib/onebox/helpers_spec.rb

@@ -24,4 +24,22 @@ RSpec.describe Onebox::Helpers do
     it { expect(described_class.truncate(test_string,100)).to eq("Chops off on spaces") }
     it { expect(described_class.truncate(" #{test_string} ",6)).to eq(" Chops...") }
   end
-end
+
+  describe "fetch_response" do
+    after(:each) do
+      Onebox.options = Onebox::DEFAULTS
+    end
+
+    before do
+      Onebox.options = { max_download_kb: 1 }
+      fake("http://example.com/large-file", response("slides"))
+    end
+
+    it "raises an exception when responses are larger than our limit" do
+      expect {
+        described_class.fetch_response('http://example.com/large-file')
+      }.to raise_error(Onebox::Helpers::DownloadTooLarge)
+    end
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onebox
 version: !ruby/object:Gem::Version
-  version: 1.7.5
+  version: 1.7.6
 platform: ruby
 authors:
 - Joanna Zeta
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-19 00:00:00.000000000 Z
+date: 2017-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -486,7 +486,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.7
 signing_key: 
 specification_version: 4
 summary: A gem for turning URLs into previews.
@@ -550,3 +550,4 @@ test_files:
 - spec/lib/onebox_spec.rb
 - spec/spec_helper.rb
 - spec/support/html_spec_helper.rb
+has_rdoc: