onebox 1.7.5 → 1.7.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9b37cb209211e4d83b000bf4143ceb893ee46c1d
4
- data.tar.gz: ff64989d2bb635abd2178b8a67e6fc47aab94aaf
3
+ metadata.gz: 9ee1f0a234b8d376cceb7543476abc508345e061
4
+ data.tar.gz: 9374c5a627bab394767c26f8efa157000b36f3b6
5
5
  SHA512:
6
- metadata.gz: 81943e75131d21db1bccf904a7549277c390b5c0a000b203a988292a55454c18100fb01c39054bec04ac5ffd51cc88cbe22fc6ab58fe2e43b50949eaeb5ad886
7
- data.tar.gz: 6125b92caca985a46e607e94b00df093840be649353191914e11b81071ea874c394860a1dc159e1862d2e6554332860839a63cea13da91d55f5bc6830b24fa0c
6
+ metadata.gz: afe7b6be89693cbca5ecf84dc97d00e904ec19ba3120f0e97252d3bfad42dd1de97a5caecc783ac2246804bfc20f1c5665b3e97cb72939ddd1ffdc9d72e05963
7
+ data.tar.gz: 76a0d2d867c9de7f0fec490af7e33b3f2294b9bfeab2ec14047d5866fd502dd9a501a7acc541ff5b271d936836e7ddea47ffb1c9e28888c1003cbee84ab49e6a
@@ -15,6 +15,7 @@ module Onebox
15
15
  cache: Moneta.new(:Memory, expires: true, serializer: :json),
16
16
  connect_timeout: 5,
17
17
  timeout: 10,
18
+ max_download_kb: (10 * 1024), # 10MB
18
19
  load_paths: [File.join(Gem::Specification.find_by_name("onebox").gem_dir, "templates")]
19
20
  }
20
21
 
@@ -45,8 +45,8 @@ module Onebox
45
45
  end
46
46
 
47
47
  def get_og_data
48
- response = Onebox::Helpers.fetch_response(url, 10)
49
- html = Nokogiri::HTML(response.body)
48
+ response = Onebox::Helpers.fetch_response(url, 10) rescue nil
49
+ html = Nokogiri::HTML(response)
50
50
  og_data = {}
51
51
  html.css('meta').each do |m|
52
52
  if m.attribute('property') && m.attribute('property').to_s.match(/^og:/i)
@@ -47,7 +47,8 @@ module Onebox
47
47
  end
48
48
 
49
49
  def is_album?
50
- oembed_data = Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response("http://api.imgur.com/oembed.json?url=#{url}").body))
50
+ response = Onebox::Helpers.fetch_response("http://api.imgur.com/oembed.json?url=#{url}") rescue "{}"
51
+ oembed_data = Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
51
52
  imgur_data_id = Nokogiri::HTML(oembed_data[:html]).xpath("//blockquote").attr("data-id")
52
53
  imgur_data_id.to_s[/a\//]
53
54
  end
@@ -29,8 +29,8 @@ module Onebox
29
29
 
30
30
  def lines
31
31
  return @lines if @lines
32
- response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1)
33
- @lines = response.body.split("\n")
32
+ response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1) rescue ""
33
+ @lines = response.split("\n")
34
34
  end
35
35
 
36
36
  def paste_key
@@ -23,7 +23,8 @@ module Onebox
23
23
  @oembed_data ||= begin
24
24
  oembed_url = "https://soundcloud.com/oembed.json?url=#{url}"
25
25
  oembed_url << "&maxheight=166" unless url["/sets/"]
26
- Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(oembed_url).body))
26
+ response = Onebox::Helpers.fetch_response(oembed_url) rescue "{}"
27
+ Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
27
28
  rescue
28
29
  {}
29
30
  end
@@ -50,7 +50,7 @@ module Onebox
50
50
  protected
51
51
 
52
52
  def html_doc
53
- @html_doc ||= Nokogiri::HTML(Onebox::Helpers.fetch_response(url).body) rescue nil
53
+ @html_doc ||= Nokogiri::HTML((Onebox::Helpers.fetch_response(url) rescue nil))
54
54
  end
55
55
 
56
56
  def get_oembed
@@ -77,7 +77,8 @@ module Onebox
77
77
 
78
78
  return {} if Onebox::Helpers.blank?(oembed_url)
79
79
 
80
- oe = Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(oembed_url).body))
80
+ json_response = Onebox::Helpers.fetch_response(oembed_url) rescue "{}"
81
+ oe = Onebox::Helpers.symbolize_keys(::MultiJson.load(json_response))
81
82
 
82
83
  # never use oembed from WordPress 4.4 (it's broken)
83
84
  oe.delete(:html) if oe[:html] && oe[:html]["wp-embedded-content"]
@@ -5,14 +5,14 @@ module Onebox
5
5
  include LayoutSupport
6
6
  include HTML
7
7
 
8
- matches_regexp Regexp.new("^https?://(?:www\\.)?(?:(?:\\w)+\\.)?(twitter)\\.com(?:/)?(?:.)*/status(es)?/")
8
+ matches_regexp /^https?:\/\/(mobile\.|www\.)?twitter\.com\/.+?\/status(es)?\/\d+$/
9
9
  always_https
10
10
 
11
11
  private
12
12
 
13
13
  def get_twitter_data
14
- response = Onebox::Helpers.fetch_response(url)
15
- html = Nokogiri::HTML(response.body)
14
+ response = Onebox::Helpers.fetch_response(url) rescue nil
15
+ html = Nokogiri::HTML(response)
16
16
  twitter_data = {}
17
17
  html.css('meta').each do |m|
18
18
  if m.attribute('property') && m.attribute('property').to_s.match(/^og:/i)
@@ -84,7 +84,8 @@ module Onebox
84
84
  def list_thumbnail_url
85
85
  @list_thumbnail_url ||= begin
86
86
  url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/playlist?list=#{list_id}"
87
- data = Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(url).body))
87
+ response = Onebox::Helpers.fetch_response(url) rescue "{}"
88
+ data = Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
88
89
  data[:thumbnail_url]
89
90
  rescue
90
91
  nil
@@ -93,12 +94,14 @@ module Onebox
93
94
 
94
95
  def video_oembed_data
95
96
  url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/watch?v=#{video_id}"
96
- Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(url).body))
97
+ response = Onebox::Helpers.fetch_response(url) rescue "{}"
98
+ Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
97
99
  end
98
100
 
99
101
  def list_oembed_data
100
102
  url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/playlist?list=#{list_id}"
101
- Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(url).body))
103
+ response = Onebox::Helpers.fetch_response(url) rescue "{}"
104
+ Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
102
105
  end
103
106
 
104
107
  def embed_params
@@ -1,5 +1,8 @@
1
1
  module Onebox
2
2
  module Helpers
3
+
4
+ class DownloadTooLarge < Exception; end;
5
+
3
6
  def self.symbolize_keys(hash)
4
7
  return {} if hash.nil?
5
8
 
@@ -15,33 +18,54 @@ module Onebox
15
18
  html.gsub(/<[^>]+>/, ' ').gsub(/\n/, '')
16
19
  end
17
20
 
18
- def self.fetch_response(location, limit = 5, domain = nil, headers = nil)
21
+ def self.fetch_response(location, limit=5, domain=nil, headers=nil)
19
22
  raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0
20
23
 
21
24
  uri = URI(location)
22
25
  uri = URI("#{domain}#{location}") if !uri.host
23
26
 
24
- http = Net::HTTP.new(uri.host, uri.port)
25
- http.open_timeout = Onebox.options.connect_timeout
26
- http.read_timeout = Onebox.options.timeout
27
- if uri.is_a?(URI::HTTPS)
28
- http.use_ssl = true
29
- http.verify_mode = OpenSSL::SSL::VERIFY_NONE
30
- end
27
+ result = StringIO.new
28
+ Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http|
29
+ http.open_timeout = Onebox.options.connect_timeout
30
+ http.read_timeout = Onebox.options.timeout
31
+ if uri.is_a?(URI::HTTPS)
32
+ http.use_ssl = true
33
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
34
+ end
31
35
 
32
- response = http.request_get(uri.request_uri,headers)
36
+ request = Net::HTTP::Get.new(uri.request_uri, headers)
37
+ start_time = Time.now
33
38
 
34
- if cookie = response.get_fields('set-cookie')
35
- header = { 'cookie' => cookie.join }
36
- end
39
+ puts Onebox.options.max_download_kb
40
+ size_bytes = Onebox.options.max_download_kb * 1024
41
+ puts "size_byes: #{size_bytes}"
42
+ http.request(request) do |response|
43
+
44
+ if cookie = response.get_fields('set-cookie')
45
+ header = { 'cookie' => cookie.join }
46
+ end
47
+
48
+ header = nil unless header.is_a? Hash
49
+
50
+ code = response.code.to_i
51
+ unless code === 200
52
+ response.error! unless [301, 302].include?(code)
53
+ return fetch_response(
54
+ response['location'],
55
+ limit - 1,
56
+ "#{uri.scheme}://#{uri.host}",
57
+ header
58
+ )
59
+ end
37
60
 
38
- header = nil unless header.is_a? Hash
61
+ response.read_body do |chunk|
62
+ result.write(chunk)
63
+ raise DownloadTooLarge.new if result.size > size_bytes
64
+ raise Timeout::Error.new if (Time.now - start_time) > Onebox.options.timeout
65
+ end
39
66
 
40
- case response
41
- when Net::HTTPSuccess then response
42
- when Net::HTTPRedirection then fetch_response(response['location'], limit - 1, "#{uri.scheme}://#{uri.host}",header)
43
- else
44
- response.error!
67
+ return result.string
68
+ end
45
69
  end
46
70
  end
47
71
 
@@ -71,7 +95,7 @@ module Onebox
71
95
  # expect properly encoded url, remove any unsafe chars
72
96
  url.gsub!("'", "&apos;")
73
97
  url.gsub!('"', "&quot;")
74
- url.gsub!(/[^\w\-`._~:\/?#\[\]@!$&'\(\)*+,;=]/, "")
98
+ url.gsub!(/[^\w\-`.~:\/?#\[\]@!$&'\(\)*+,;=]/, "")
75
99
  url
76
100
  end
77
101
 
@@ -1,3 +1,3 @@
1
1
  module Onebox
2
- VERSION = "1.7.5"
2
+ VERSION = "1.7.6"
3
3
  end
@@ -84,19 +84,23 @@ describe Onebox::Engine::WhitelistedGenericOnebox do
84
84
  end
85
85
 
86
86
  describe 'to_html' do
87
-
88
87
  before do
89
88
  described_class.whitelist = %w(dailymail.co.uk discourse.org)
90
89
  original_link = "http://www.dailymail.co.uk/pages/live/articles/news/news.html?in_article_id=479146&in_page_id=1770"
91
90
  redirect_link = 'http://www.dailymail.co.uk/news/article-479146/Brutality-justice-The-truth-tarred-feathered-drug-dealer.html'
92
- FakeWeb.register_uri(:get, original_link, status: ["301", "Moved Permanently"], location: '/news/article-479146/Brutality-justice-The-truth-tarred-feathered-drug-dealer.html')
91
+ FakeWeb.register_uri(
92
+ :get,
93
+ original_link,
94
+ status: ["301", "Moved Permanently"],
95
+ location: redirect_link
96
+ )
93
97
  fake(redirect_link, response('dailymail'))
94
98
  onebox = described_class.new(original_link)
95
99
  @html = onebox.to_html
96
100
  end
97
101
  let(:html) { @html }
98
102
 
99
- it "includes summary" do
103
+ it "follows redirects and includes the summary" do
100
104
  expect(html).to include("It was the most chilling image of the week")
101
105
  end
102
106
  end
@@ -24,4 +24,22 @@ RSpec.describe Onebox::Helpers do
24
24
  it { expect(described_class.truncate(test_string,100)).to eq("Chops off on spaces") }
25
25
  it { expect(described_class.truncate(" #{test_string} ",6)).to eq(" Chops...") }
26
26
  end
27
- end
27
+
28
+ describe "fetch_response" do
29
+ after(:each) do
30
+ Onebox.options = Onebox::DEFAULTS
31
+ end
32
+
33
+ before do
34
+ Onebox.options = { max_download_kb: 1 }
35
+ fake("http://example.com/large-file", response("slides"))
36
+ end
37
+
38
+ it "raises an exception when responses are larger than our limit" do
39
+ expect {
40
+ described_class.fetch_response('http://example.com/large-file')
41
+ }.to raise_error(Onebox::Helpers::DownloadTooLarge)
42
+ end
43
+ end
44
+
45
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: onebox
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.5
4
+ version: 1.7.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Joanna Zeta
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2017-01-19 00:00:00.000000000 Z
13
+ date: 2017-01-25 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: multi_json
@@ -486,7 +486,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
486
486
  version: '0'
487
487
  requirements: []
488
488
  rubyforge_project:
489
- rubygems_version: 2.6.8
489
+ rubygems_version: 2.6.7
490
490
  signing_key:
491
491
  specification_version: 4
492
492
  summary: A gem for turning URLs into previews.
@@ -550,3 +550,4 @@ test_files:
550
550
  - spec/lib/onebox_spec.rb
551
551
  - spec/spec_helper.rb
552
552
  - spec/support/html_spec_helper.rb
553
+ has_rdoc: