onebox 1.7.5 → 1.7.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9b37cb209211e4d83b000bf4143ceb893ee46c1d
4
- data.tar.gz: ff64989d2bb635abd2178b8a67e6fc47aab94aaf
3
+ metadata.gz: 9ee1f0a234b8d376cceb7543476abc508345e061
4
+ data.tar.gz: 9374c5a627bab394767c26f8efa157000b36f3b6
5
5
  SHA512:
6
- metadata.gz: 81943e75131d21db1bccf904a7549277c390b5c0a000b203a988292a55454c18100fb01c39054bec04ac5ffd51cc88cbe22fc6ab58fe2e43b50949eaeb5ad886
7
- data.tar.gz: 6125b92caca985a46e607e94b00df093840be649353191914e11b81071ea874c394860a1dc159e1862d2e6554332860839a63cea13da91d55f5bc6830b24fa0c
6
+ metadata.gz: afe7b6be89693cbca5ecf84dc97d00e904ec19ba3120f0e97252d3bfad42dd1de97a5caecc783ac2246804bfc20f1c5665b3e97cb72939ddd1ffdc9d72e05963
7
+ data.tar.gz: 76a0d2d867c9de7f0fec490af7e33b3f2294b9bfeab2ec14047d5866fd502dd9a501a7acc541ff5b271d936836e7ddea47ffb1c9e28888c1003cbee84ab49e6a
@@ -15,6 +15,7 @@ module Onebox
15
15
  cache: Moneta.new(:Memory, expires: true, serializer: :json),
16
16
  connect_timeout: 5,
17
17
  timeout: 10,
18
+ max_download_kb: (10 * 1024), # 10MB
18
19
  load_paths: [File.join(Gem::Specification.find_by_name("onebox").gem_dir, "templates")]
19
20
  }
20
21
 
@@ -45,8 +45,8 @@ module Onebox
45
45
  end
46
46
 
47
47
  def get_og_data
48
- response = Onebox::Helpers.fetch_response(url, 10)
49
- html = Nokogiri::HTML(response.body)
48
+ response = Onebox::Helpers.fetch_response(url, 10) rescue nil
49
+ html = Nokogiri::HTML(response)
50
50
  og_data = {}
51
51
  html.css('meta').each do |m|
52
52
  if m.attribute('property') && m.attribute('property').to_s.match(/^og:/i)
@@ -47,7 +47,8 @@ module Onebox
47
47
  end
48
48
 
49
49
  def is_album?
50
- oembed_data = Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response("http://api.imgur.com/oembed.json?url=#{url}").body))
50
+ response = Onebox::Helpers.fetch_response("http://api.imgur.com/oembed.json?url=#{url}") rescue "{}"
51
+ oembed_data = Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
51
52
  imgur_data_id = Nokogiri::HTML(oembed_data[:html]).xpath("//blockquote").attr("data-id")
52
53
  imgur_data_id.to_s[/a\//]
53
54
  end
@@ -29,8 +29,8 @@ module Onebox
29
29
 
30
30
  def lines
31
31
  return @lines if @lines
32
- response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1)
33
- @lines = response.body.split("\n")
32
+ response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1) rescue ""
33
+ @lines = response.split("\n")
34
34
  end
35
35
 
36
36
  def paste_key
@@ -23,7 +23,8 @@ module Onebox
23
23
  @oembed_data ||= begin
24
24
  oembed_url = "https://soundcloud.com/oembed.json?url=#{url}"
25
25
  oembed_url << "&maxheight=166" unless url["/sets/"]
26
- Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(oembed_url).body))
26
+ response = Onebox::Helpers.fetch_response(oembed_url) rescue "{}"
27
+ Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
27
28
  rescue
28
29
  {}
29
30
  end
@@ -50,7 +50,7 @@ module Onebox
50
50
  protected
51
51
 
52
52
  def html_doc
53
- @html_doc ||= Nokogiri::HTML(Onebox::Helpers.fetch_response(url).body) rescue nil
53
+ @html_doc ||= Nokogiri::HTML((Onebox::Helpers.fetch_response(url) rescue nil))
54
54
  end
55
55
 
56
56
  def get_oembed
@@ -77,7 +77,8 @@ module Onebox
77
77
 
78
78
  return {} if Onebox::Helpers.blank?(oembed_url)
79
79
 
80
- oe = Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(oembed_url).body))
80
+ json_response = Onebox::Helpers.fetch_response(oembed_url) rescue "{}"
81
+ oe = Onebox::Helpers.symbolize_keys(::MultiJson.load(json_response))
81
82
 
82
83
  # never use oembed from WordPress 4.4 (it's broken)
83
84
  oe.delete(:html) if oe[:html] && oe[:html]["wp-embedded-content"]
@@ -5,14 +5,14 @@ module Onebox
5
5
  include LayoutSupport
6
6
  include HTML
7
7
 
8
- matches_regexp Regexp.new("^https?://(?:www\\.)?(?:(?:\\w)+\\.)?(twitter)\\.com(?:/)?(?:.)*/status(es)?/")
8
+ matches_regexp /^https?:\/\/(mobile\.|www\.)?twitter\.com\/.+?\/status(es)?\/\d+$/
9
9
  always_https
10
10
 
11
11
  private
12
12
 
13
13
  def get_twitter_data
14
- response = Onebox::Helpers.fetch_response(url)
15
- html = Nokogiri::HTML(response.body)
14
+ response = Onebox::Helpers.fetch_response(url) rescue nil
15
+ html = Nokogiri::HTML(response)
16
16
  twitter_data = {}
17
17
  html.css('meta').each do |m|
18
18
  if m.attribute('property') && m.attribute('property').to_s.match(/^og:/i)
@@ -84,7 +84,8 @@ module Onebox
84
84
  def list_thumbnail_url
85
85
  @list_thumbnail_url ||= begin
86
86
  url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/playlist?list=#{list_id}"
87
- data = Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(url).body))
87
+ response = Onebox::Helpers.fetch_response(url) rescue "{}"
88
+ data = Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
88
89
  data[:thumbnail_url]
89
90
  rescue
90
91
  nil
@@ -93,12 +94,14 @@ module Onebox
93
94
 
94
95
  def video_oembed_data
95
96
  url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/watch?v=#{video_id}"
96
- Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(url).body))
97
+ response = Onebox::Helpers.fetch_response(url) rescue "{}"
98
+ Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
97
99
  end
98
100
 
99
101
  def list_oembed_data
100
102
  url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/playlist?list=#{list_id}"
101
- Onebox::Helpers.symbolize_keys(::MultiJson.load(Onebox::Helpers.fetch_response(url).body))
103
+ response = Onebox::Helpers.fetch_response(url) rescue "{}"
104
+ Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
102
105
  end
103
106
 
104
107
  def embed_params
@@ -1,5 +1,8 @@
1
1
  module Onebox
2
2
  module Helpers
3
+
4
+ class DownloadTooLarge < Exception; end;
5
+
3
6
  def self.symbolize_keys(hash)
4
7
  return {} if hash.nil?
5
8
 
@@ -15,33 +18,54 @@ module Onebox
15
18
  html.gsub(/<[^>]+>/, ' ').gsub(/\n/, '')
16
19
  end
17
20
 
18
- def self.fetch_response(location, limit = 5, domain = nil, headers = nil)
21
+ def self.fetch_response(location, limit=5, domain=nil, headers=nil)
19
22
  raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0
20
23
 
21
24
  uri = URI(location)
22
25
  uri = URI("#{domain}#{location}") if !uri.host
23
26
 
24
- http = Net::HTTP.new(uri.host, uri.port)
25
- http.open_timeout = Onebox.options.connect_timeout
26
- http.read_timeout = Onebox.options.timeout
27
- if uri.is_a?(URI::HTTPS)
28
- http.use_ssl = true
29
- http.verify_mode = OpenSSL::SSL::VERIFY_NONE
30
- end
27
+ result = StringIO.new
28
+ Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http|
29
+ http.open_timeout = Onebox.options.connect_timeout
30
+ http.read_timeout = Onebox.options.timeout
31
+ if uri.is_a?(URI::HTTPS)
32
+ http.use_ssl = true
33
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
34
+ end
31
35
 
32
- response = http.request_get(uri.request_uri,headers)
36
+ request = Net::HTTP::Get.new(uri.request_uri, headers)
37
+ start_time = Time.now
33
38
 
34
- if cookie = response.get_fields('set-cookie')
35
- header = { 'cookie' => cookie.join }
36
- end
39
+ puts Onebox.options.max_download_kb
40
+ size_bytes = Onebox.options.max_download_kb * 1024
41
+ puts "size_byes: #{size_bytes}"
42
+ http.request(request) do |response|
43
+
44
+ if cookie = response.get_fields('set-cookie')
45
+ header = { 'cookie' => cookie.join }
46
+ end
47
+
48
+ header = nil unless header.is_a? Hash
49
+
50
+ code = response.code.to_i
51
+ unless code === 200
52
+ response.error! unless [301, 302].include?(code)
53
+ return fetch_response(
54
+ response['location'],
55
+ limit - 1,
56
+ "#{uri.scheme}://#{uri.host}",
57
+ header
58
+ )
59
+ end
37
60
 
38
- header = nil unless header.is_a? Hash
61
+ response.read_body do |chunk|
62
+ result.write(chunk)
63
+ raise DownloadTooLarge.new if result.size > size_bytes
64
+ raise Timeout::Error.new if (Time.now - start_time) > Onebox.options.timeout
65
+ end
39
66
 
40
- case response
41
- when Net::HTTPSuccess then response
42
- when Net::HTTPRedirection then fetch_response(response['location'], limit - 1, "#{uri.scheme}://#{uri.host}",header)
43
- else
44
- response.error!
67
+ return result.string
68
+ end
45
69
  end
46
70
  end
47
71
 
@@ -71,7 +95,7 @@ module Onebox
71
95
  # expect properly encoded url, remove any unsafe chars
72
96
  url.gsub!("'", "&apos;")
73
97
  url.gsub!('"', "&quot;")
74
- url.gsub!(/[^\w\-`._~:\/?#\[\]@!$&'\(\)*+,;=]/, "")
98
+ url.gsub!(/[^\w\-`.~:\/?#\[\]@!$&'\(\)*+,;=]/, "")
75
99
  url
76
100
  end
77
101
 
@@ -1,3 +1,3 @@
1
1
  module Onebox
2
- VERSION = "1.7.5"
2
+ VERSION = "1.7.6"
3
3
  end
@@ -84,19 +84,23 @@ describe Onebox::Engine::WhitelistedGenericOnebox do
84
84
  end
85
85
 
86
86
  describe 'to_html' do
87
-
88
87
  before do
89
88
  described_class.whitelist = %w(dailymail.co.uk discourse.org)
90
89
  original_link = "http://www.dailymail.co.uk/pages/live/articles/news/news.html?in_article_id=479146&in_page_id=1770"
91
90
  redirect_link = 'http://www.dailymail.co.uk/news/article-479146/Brutality-justice-The-truth-tarred-feathered-drug-dealer.html'
92
- FakeWeb.register_uri(:get, original_link, status: ["301", "Moved Permanently"], location: '/news/article-479146/Brutality-justice-The-truth-tarred-feathered-drug-dealer.html')
91
+ FakeWeb.register_uri(
92
+ :get,
93
+ original_link,
94
+ status: ["301", "Moved Permanently"],
95
+ location: redirect_link
96
+ )
93
97
  fake(redirect_link, response('dailymail'))
94
98
  onebox = described_class.new(original_link)
95
99
  @html = onebox.to_html
96
100
  end
97
101
  let(:html) { @html }
98
102
 
99
- it "includes summary" do
103
+ it "follows redirects and includes the summary" do
100
104
  expect(html).to include("It was the most chilling image of the week")
101
105
  end
102
106
  end
@@ -24,4 +24,22 @@ RSpec.describe Onebox::Helpers do
24
24
  it { expect(described_class.truncate(test_string,100)).to eq("Chops off on spaces") }
25
25
  it { expect(described_class.truncate(" #{test_string} ",6)).to eq(" Chops...") }
26
26
  end
27
- end
27
+
28
+ describe "fetch_response" do
29
+ after(:each) do
30
+ Onebox.options = Onebox::DEFAULTS
31
+ end
32
+
33
+ before do
34
+ Onebox.options = { max_download_kb: 1 }
35
+ fake("http://example.com/large-file", response("slides"))
36
+ end
37
+
38
+ it "raises an exception when responses are larger than our limit" do
39
+ expect {
40
+ described_class.fetch_response('http://example.com/large-file')
41
+ }.to raise_error(Onebox::Helpers::DownloadTooLarge)
42
+ end
43
+ end
44
+
45
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: onebox
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.5
4
+ version: 1.7.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Joanna Zeta
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2017-01-19 00:00:00.000000000 Z
13
+ date: 2017-01-25 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: multi_json
@@ -486,7 +486,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
486
486
  version: '0'
487
487
  requirements: []
488
488
  rubyforge_project:
489
- rubygems_version: 2.6.8
489
+ rubygems_version: 2.6.7
490
490
  signing_key:
491
491
  specification_version: 4
492
492
  summary: A gem for turning URLs into previews.
@@ -550,3 +550,4 @@ test_files:
550
550
  - spec/lib/onebox_spec.rb
551
551
  - spec/spec_helper.rb
552
552
  - spec/support/html_spec_helper.rb
553
+ has_rdoc: