onebox 1.7.5 → 1.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/onebox.rb +1 -0
- data/lib/onebox/engine/google_docs_onebox.rb +2 -2
- data/lib/onebox/engine/imgur_onebox.rb +2 -1
- data/lib/onebox/engine/pastebin_onebox.rb +2 -2
- data/lib/onebox/engine/soundcloud_onebox.rb +2 -1
- data/lib/onebox/engine/standard_embed.rb +3 -2
- data/lib/onebox/engine/twitter_status_onebox.rb +3 -3
- data/lib/onebox/engine/youtube_onebox.rb +6 -3
- data/lib/onebox/helpers.rb +43 -19
- data/lib/onebox/version.rb +1 -1
- data/spec/lib/onebox/engine/whitelisted_generic_onebox_spec.rb +7 -3
- data/spec/lib/onebox/helpers_spec.rb +19 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9ee1f0a234b8d376cceb7543476abc508345e061
|
4
|
+
data.tar.gz: 9374c5a627bab394767c26f8efa157000b36f3b6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: afe7b6be89693cbca5ecf84dc97d00e904ec19ba3120f0e97252d3bfad42dd1de97a5caecc783ac2246804bfc20f1c5665b3e97cb72939ddd1ffdc9d72e05963
|
7
|
+
data.tar.gz: 76a0d2d867c9de7f0fec490af7e33b3f2294b9bfeab2ec14047d5866fd502dd9a501a7acc541ff5b271d936836e7ddea47ffb1c9e28888c1003cbee84ab49e6a
|
data/lib/onebox.rb
CHANGED
@@ -45,8 +45,8 @@ module Onebox
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def get_og_data
|
48
|
-
response = Onebox::Helpers.fetch_response(url, 10)
|
49
|
-
html = Nokogiri::HTML(response
|
48
|
+
response = Onebox::Helpers.fetch_response(url, 10) rescue nil
|
49
|
+
html = Nokogiri::HTML(response)
|
50
50
|
og_data = {}
|
51
51
|
html.css('meta').each do |m|
|
52
52
|
if m.attribute('property') && m.attribute('property').to_s.match(/^og:/i)
|
@@ -47,7 +47,8 @@ module Onebox
|
|
47
47
|
end
|
48
48
|
|
49
49
|
def is_album?
|
50
|
-
|
50
|
+
response = Onebox::Helpers.fetch_response("http://api.imgur.com/oembed.json?url=#{url}") rescue "{}"
|
51
|
+
oembed_data = Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
|
51
52
|
imgur_data_id = Nokogiri::HTML(oembed_data[:html]).xpath("//blockquote").attr("data-id")
|
52
53
|
imgur_data_id.to_s[/a\//]
|
53
54
|
end
|
@@ -29,8 +29,8 @@ module Onebox
|
|
29
29
|
|
30
30
|
def lines
|
31
31
|
return @lines if @lines
|
32
|
-
response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1)
|
33
|
-
@lines = response.
|
32
|
+
response = Onebox::Helpers.fetch_response("http://pastebin.com/raw/#{paste_key}", 1) rescue ""
|
33
|
+
@lines = response.split("\n")
|
34
34
|
end
|
35
35
|
|
36
36
|
def paste_key
|
@@ -23,7 +23,8 @@ module Onebox
|
|
23
23
|
@oembed_data ||= begin
|
24
24
|
oembed_url = "https://soundcloud.com/oembed.json?url=#{url}"
|
25
25
|
oembed_url << "&maxheight=166" unless url["/sets/"]
|
26
|
-
Onebox::Helpers.
|
26
|
+
response = Onebox::Helpers.fetch_response(oembed_url) rescue "{}"
|
27
|
+
Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
|
27
28
|
rescue
|
28
29
|
{}
|
29
30
|
end
|
@@ -50,7 +50,7 @@ module Onebox
|
|
50
50
|
protected
|
51
51
|
|
52
52
|
def html_doc
|
53
|
-
@html_doc ||= Nokogiri::HTML(Onebox::Helpers.fetch_response(url)
|
53
|
+
@html_doc ||= Nokogiri::HTML((Onebox::Helpers.fetch_response(url) rescue nil))
|
54
54
|
end
|
55
55
|
|
56
56
|
def get_oembed
|
@@ -77,7 +77,8 @@ module Onebox
|
|
77
77
|
|
78
78
|
return {} if Onebox::Helpers.blank?(oembed_url)
|
79
79
|
|
80
|
-
|
80
|
+
json_response = Onebox::Helpers.fetch_response(oembed_url) rescue "{}"
|
81
|
+
oe = Onebox::Helpers.symbolize_keys(::MultiJson.load(json_response))
|
81
82
|
|
82
83
|
# never use oembed from WordPress 4.4 (it's broken)
|
83
84
|
oe.delete(:html) if oe[:html] && oe[:html]["wp-embedded-content"]
|
@@ -5,14 +5,14 @@ module Onebox
|
|
5
5
|
include LayoutSupport
|
6
6
|
include HTML
|
7
7
|
|
8
|
-
matches_regexp
|
8
|
+
matches_regexp /^https?:\/\/(mobile\.|www\.)?twitter\.com\/.+?\/status(es)?\/\d+$/
|
9
9
|
always_https
|
10
10
|
|
11
11
|
private
|
12
12
|
|
13
13
|
def get_twitter_data
|
14
|
-
response = Onebox::Helpers.fetch_response(url)
|
15
|
-
html = Nokogiri::HTML(response
|
14
|
+
response = Onebox::Helpers.fetch_response(url) rescue nil
|
15
|
+
html = Nokogiri::HTML(response)
|
16
16
|
twitter_data = {}
|
17
17
|
html.css('meta').each do |m|
|
18
18
|
if m.attribute('property') && m.attribute('property').to_s.match(/^og:/i)
|
@@ -84,7 +84,8 @@ module Onebox
|
|
84
84
|
def list_thumbnail_url
|
85
85
|
@list_thumbnail_url ||= begin
|
86
86
|
url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/playlist?list=#{list_id}"
|
87
|
-
|
87
|
+
response = Onebox::Helpers.fetch_response(url) rescue "{}"
|
88
|
+
data = Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
|
88
89
|
data[:thumbnail_url]
|
89
90
|
rescue
|
90
91
|
nil
|
@@ -93,12 +94,14 @@ module Onebox
|
|
93
94
|
|
94
95
|
def video_oembed_data
|
95
96
|
url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/watch?v=#{video_id}"
|
96
|
-
Onebox::Helpers.
|
97
|
+
response = Onebox::Helpers.fetch_response(url) rescue "{}"
|
98
|
+
Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
|
97
99
|
end
|
98
100
|
|
99
101
|
def list_oembed_data
|
100
102
|
url = "https://www.youtube.com/oembed?format=json&url=https://www.youtube.com/playlist?list=#{list_id}"
|
101
|
-
Onebox::Helpers.
|
103
|
+
response = Onebox::Helpers.fetch_response(url) rescue "{}"
|
104
|
+
Onebox::Helpers.symbolize_keys(::MultiJson.load(response))
|
102
105
|
end
|
103
106
|
|
104
107
|
def embed_params
|
data/lib/onebox/helpers.rb
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
module Onebox
|
2
2
|
module Helpers
|
3
|
+
|
4
|
+
class DownloadTooLarge < Exception; end;
|
5
|
+
|
3
6
|
def self.symbolize_keys(hash)
|
4
7
|
return {} if hash.nil?
|
5
8
|
|
@@ -15,33 +18,54 @@ module Onebox
|
|
15
18
|
html.gsub(/<[^>]+>/, ' ').gsub(/\n/, '')
|
16
19
|
end
|
17
20
|
|
18
|
-
def self.fetch_response(location, limit
|
21
|
+
def self.fetch_response(location, limit=5, domain=nil, headers=nil)
|
19
22
|
raise Net::HTTPError.new('HTTP redirect too deep', location) if limit == 0
|
20
23
|
|
21
24
|
uri = URI(location)
|
22
25
|
uri = URI("#{domain}#{location}") if !uri.host
|
23
26
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
27
|
+
result = StringIO.new
|
28
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http|
|
29
|
+
http.open_timeout = Onebox.options.connect_timeout
|
30
|
+
http.read_timeout = Onebox.options.timeout
|
31
|
+
if uri.is_a?(URI::HTTPS)
|
32
|
+
http.use_ssl = true
|
33
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
34
|
+
end
|
31
35
|
|
32
|
-
|
36
|
+
request = Net::HTTP::Get.new(uri.request_uri, headers)
|
37
|
+
start_time = Time.now
|
33
38
|
|
34
|
-
|
35
|
-
|
36
|
-
|
39
|
+
puts Onebox.options.max_download_kb
|
40
|
+
size_bytes = Onebox.options.max_download_kb * 1024
|
41
|
+
puts "size_byes: #{size_bytes}"
|
42
|
+
http.request(request) do |response|
|
43
|
+
|
44
|
+
if cookie = response.get_fields('set-cookie')
|
45
|
+
header = { 'cookie' => cookie.join }
|
46
|
+
end
|
47
|
+
|
48
|
+
header = nil unless header.is_a? Hash
|
49
|
+
|
50
|
+
code = response.code.to_i
|
51
|
+
unless code === 200
|
52
|
+
response.error! unless [301, 302].include?(code)
|
53
|
+
return fetch_response(
|
54
|
+
response['location'],
|
55
|
+
limit - 1,
|
56
|
+
"#{uri.scheme}://#{uri.host}",
|
57
|
+
header
|
58
|
+
)
|
59
|
+
end
|
37
60
|
|
38
|
-
|
61
|
+
response.read_body do |chunk|
|
62
|
+
result.write(chunk)
|
63
|
+
raise DownloadTooLarge.new if result.size > size_bytes
|
64
|
+
raise Timeout::Error.new if (Time.now - start_time) > Onebox.options.timeout
|
65
|
+
end
|
39
66
|
|
40
|
-
|
41
|
-
|
42
|
-
when Net::HTTPRedirection then fetch_response(response['location'], limit - 1, "#{uri.scheme}://#{uri.host}",header)
|
43
|
-
else
|
44
|
-
response.error!
|
67
|
+
return result.string
|
68
|
+
end
|
45
69
|
end
|
46
70
|
end
|
47
71
|
|
@@ -71,7 +95,7 @@ module Onebox
|
|
71
95
|
# expect properly encoded url, remove any unsafe chars
|
72
96
|
url.gsub!("'", "'")
|
73
97
|
url.gsub!('"', """)
|
74
|
-
url.gsub!(/[^\w
|
98
|
+
url.gsub!(/[^\w\-`.~:\/?#\[\]@!$&'\(\)*+,;=]/, "")
|
75
99
|
url
|
76
100
|
end
|
77
101
|
|
data/lib/onebox/version.rb
CHANGED
@@ -84,19 +84,23 @@ describe Onebox::Engine::WhitelistedGenericOnebox do
|
|
84
84
|
end
|
85
85
|
|
86
86
|
describe 'to_html' do
|
87
|
-
|
88
87
|
before do
|
89
88
|
described_class.whitelist = %w(dailymail.co.uk discourse.org)
|
90
89
|
original_link = "http://www.dailymail.co.uk/pages/live/articles/news/news.html?in_article_id=479146&in_page_id=1770"
|
91
90
|
redirect_link = 'http://www.dailymail.co.uk/news/article-479146/Brutality-justice-The-truth-tarred-feathered-drug-dealer.html'
|
92
|
-
FakeWeb.register_uri(
|
91
|
+
FakeWeb.register_uri(
|
92
|
+
:get,
|
93
|
+
original_link,
|
94
|
+
status: ["301", "Moved Permanently"],
|
95
|
+
location: redirect_link
|
96
|
+
)
|
93
97
|
fake(redirect_link, response('dailymail'))
|
94
98
|
onebox = described_class.new(original_link)
|
95
99
|
@html = onebox.to_html
|
96
100
|
end
|
97
101
|
let(:html) { @html }
|
98
102
|
|
99
|
-
it "includes summary" do
|
103
|
+
it "follows redirects and includes the summary" do
|
100
104
|
expect(html).to include("It was the most chilling image of the week")
|
101
105
|
end
|
102
106
|
end
|
@@ -24,4 +24,22 @@ RSpec.describe Onebox::Helpers do
|
|
24
24
|
it { expect(described_class.truncate(test_string,100)).to eq("Chops off on spaces") }
|
25
25
|
it { expect(described_class.truncate(" #{test_string} ",6)).to eq(" Chops...") }
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
|
+
describe "fetch_response" do
|
29
|
+
after(:each) do
|
30
|
+
Onebox.options = Onebox::DEFAULTS
|
31
|
+
end
|
32
|
+
|
33
|
+
before do
|
34
|
+
Onebox.options = { max_download_kb: 1 }
|
35
|
+
fake("http://example.com/large-file", response("slides"))
|
36
|
+
end
|
37
|
+
|
38
|
+
it "raises an exception when responses are larger than our limit" do
|
39
|
+
expect {
|
40
|
+
described_class.fetch_response('http://example.com/large-file')
|
41
|
+
}.to raise_error(Onebox::Helpers::DownloadTooLarge)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: onebox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.7.
|
4
|
+
version: 1.7.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Joanna Zeta
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-01-
|
13
|
+
date: 2017-01-25 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: multi_json
|
@@ -486,7 +486,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
486
486
|
version: '0'
|
487
487
|
requirements: []
|
488
488
|
rubyforge_project:
|
489
|
-
rubygems_version: 2.6.
|
489
|
+
rubygems_version: 2.6.7
|
490
490
|
signing_key:
|
491
491
|
specification_version: 4
|
492
492
|
summary: A gem for turning URLs into previews.
|
@@ -550,3 +550,4 @@ test_files:
|
|
550
550
|
- spec/lib/onebox_spec.rb
|
551
551
|
- spec/spec_helper.rb
|
552
552
|
- spec/support/html_spec_helper.rb
|
553
|
+
has_rdoc:
|