link_thumbnailer 1.1.1 → 1.1.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,41 +1,41 @@
1
- # Use this hook to configure LinkThumbnailer bahaviors.
2
- LinkThumbnailer.configure do |config|
3
- # Set mandatory attributes require for the website to be valid.
4
- # You can set `strict` to false if you want to skip this validation.
5
- # config.mandatory_attributes = %w(url title image)
6
-
7
- # Whether you want to validate given website against mandatory attributes or not.
8
- # config.strict = true
9
-
10
- # Numbers of redirects before raising an exception when trying to parse given url.
11
- # config.redirect_limit = 3
12
-
13
- # List of blacklisted urls you want to skip when searching for images.
14
- # config.blacklist_urls = [
15
- # %r{^http://ad\.doubleclick\.net/},
16
- # %r{^http://b\.scorecardresearch\.com/},
17
- # %r{^http://pixel\.quantserve\.com/},
18
- # %r{^http://s7\.addthis\.com/}
19
- # ]
20
-
21
- # Included Rmagick attributes for images. See http://www.imagemagick.org/RMagick/doc/
22
- # for more details.
23
- # 'source_url' is a custom attribute and should always be included since this
24
- # is where you'll find the image url.
25
- # config.image_attributes = %w(source_url size type)
26
-
27
- # Fetch 10 images maximum.
28
- # config.limit = 10
29
-
30
- # Return top 5 images only.
31
- # config.top = 5
32
-
33
- # Set user agent
34
- # config.user_agent = 'linkthumbnailer'
35
-
36
- # Enable or disable SSL verification
37
- # config.verify_ssl = true
38
-
39
- # HTTP open_timeout: The amount of time in seconds to wait for a connection to be opened.
40
- # config.http_timeout = 5
41
- end
1
+ # Use this hook to configure LinkThumbnailer bahaviors.
2
+ LinkThumbnailer.configure do |config|
3
+ # Set mandatory attributes require for the website to be valid.
4
+ # You can set `strict` to false if you want to skip this validation.
5
+ # config.mandatory_attributes = %w(url title image)
6
+
7
+ # Whether you want to validate given website against mandatory attributes or not.
8
+ # config.strict = true
9
+
10
+ # Numbers of redirects before raising an exception when trying to parse given url.
11
+ # config.redirect_limit = 3
12
+
13
+ # List of blacklisted urls you want to skip when searching for images.
14
+ # config.blacklist_urls = [
15
+ # %r{^http://ad\.doubleclick\.net/},
16
+ # %r{^http://b\.scorecardresearch\.com/},
17
+ # %r{^http://pixel\.quantserve\.com/},
18
+ # %r{^http://s7\.addthis\.com/}
19
+ # ]
20
+
21
+ # Included Rmagick attributes for images. See http://www.imagemagick.org/RMagick/doc/
22
+ # for more details.
23
+ # 'source_url' is a custom attribute and should always be included since this
24
+ # is where you'll find the image url.
25
+ # config.image_attributes = %w(source_url size type)
26
+
27
+ # Fetch 10 images maximum.
28
+ # config.limit = 10
29
+
30
+ # Return top 5 images only.
31
+ # config.top = 5
32
+
33
+ # Set user agent
34
+ # config.user_agent = 'linkthumbnailer'
35
+
36
+ # Enable or disable SSL verification
37
+ # config.verify_ssl = true
38
+
39
+ # HTTP open_timeout: The amount of time in seconds to wait for a connection to be opened.
40
+ # config.http_timeout = 5
41
+ end
@@ -1,17 +1,17 @@
1
- require 'hashie'
2
-
3
- module LinkThumbnailer
4
- class Configuration < Hashie::Mash
5
-
6
- def rmagick_attributes
7
- LinkThumbnailer.logger.info "[DEPRECATION] rmagick_attributes is deprecated. Use image_attributes instead."
8
- image_attributes
9
- end
10
-
11
- def rmagick_attributes=(value)
12
- LinkThumbnailer.logger.info "[DEPRECATION] rmagick_attributes is deprecated. Use image_attributes instead."
13
- self.image_attributes = value
14
- end
15
-
16
- end
17
- end
1
+ require 'hashie'
2
+
3
+ module LinkThumbnailer
4
+ class Configuration < Hashie::Mash
5
+
6
+ def rmagick_attributes
7
+ LinkThumbnailer.logger.info "[DEPRECATION] rmagick_attributes is deprecated. Use image_attributes instead."
8
+ image_attributes
9
+ end
10
+
11
+ def rmagick_attributes=(value)
12
+ LinkThumbnailer.logger.info "[DEPRECATION] rmagick_attributes is deprecated. Use image_attributes instead."
13
+ self.image_attributes = value
14
+ end
15
+
16
+ end
17
+ end
@@ -1,65 +1,65 @@
1
- require 'uri'
2
-
3
- module LinkThumbnailer
4
-
5
- module Doc
6
-
7
- def doc_base_href
8
- base = at('//head/base')
9
- base['href'] if base
10
- end
11
-
12
- def img_srcs
13
- search('//img').map { |i| i['src'] }.compact
14
- end
15
-
16
- def img_abs_urls(base_url = nil)
17
- result = []
18
-
19
- img_srcs.each do |i|
20
- begin
21
- u = URI(i)
22
- rescue URI::InvalidURIError
23
- next
24
- end
25
-
26
- result << if u.is_a?(URI::HTTP)
27
- u
28
- else
29
- URI.join(base_url || doc_base_href || source_url, i)
30
- end
31
- end
32
-
33
- result
34
- end
35
-
36
- def title
37
- css('title').text.strip
38
- end
39
-
40
- def description
41
- if element = xpath("//meta[translate(@name,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz') = 'description' and @content]").first
42
- return element.attributes['content'].value.strip
43
- end
44
-
45
- css('body p').each do |node|
46
- if !node.has_attribute?('style') && node.first_element_child.nil?
47
- return node.text.strip
48
- end
49
- end
50
-
51
- nil
52
- end
53
-
54
- def canonical_url
55
- if element = xpath("//link[translate(@rel, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz') = 'canonical' and @href]").first
56
- return element.attributes['href'].value.strip
57
- end
58
- nil
59
- end
60
-
61
- attr_accessor :source_url
62
-
63
- end
64
-
65
- end
1
+ require 'uri'
2
+
3
+ module LinkThumbnailer
4
+
5
+ module Doc
6
+
7
+ def doc_base_href
8
+ base = at('//head/base')
9
+ base['href'] if base
10
+ end
11
+
12
+ def img_srcs
13
+ search('//img').map { |i| i['src'] }.compact
14
+ end
15
+
16
+ def img_abs_urls(base_url = nil)
17
+ result = []
18
+
19
+ img_srcs.each do |i|
20
+ begin
21
+ u = URI(i)
22
+ rescue URI::InvalidURIError
23
+ next
24
+ end
25
+
26
+ result << if u.is_a?(URI::HTTP)
27
+ u
28
+ else
29
+ URI.join(base_url || doc_base_href || source_url, i)
30
+ end
31
+ end
32
+
33
+ result
34
+ end
35
+
36
+ def title
37
+ css('title').text.strip
38
+ end
39
+
40
+ def description
41
+ if element = xpath("//meta[translate(@name,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz') = 'description' and @content]").first
42
+ return element.attributes['content'].value.strip
43
+ end
44
+
45
+ css('body p').each do |node|
46
+ if !node.has_attribute?('style') && node.first_element_child.nil?
47
+ return node.text.strip
48
+ end
49
+ end
50
+
51
+ nil
52
+ end
53
+
54
+ def canonical_url
55
+ if element = xpath("//link[translate(@rel, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz') = 'canonical' and @href]").first
56
+ return element.attributes['href'].value.strip
57
+ end
58
+ nil
59
+ end
60
+
61
+ attr_accessor :source_url
62
+
63
+ end
64
+
65
+ end
@@ -1,15 +1,15 @@
1
- require 'nokogiri'
2
-
3
- module LinkThumbnailer
4
-
5
- class DocParser
6
-
7
- def parse(doc_string, source_url = nil)
8
- doc = Nokogiri::HTML(doc_string).extend(LinkThumbnailer::Doc)
9
- doc.source_url = source_url
10
- doc
11
- end
12
-
13
- end
14
-
15
- end
1
+ require 'nokogiri'
2
+
3
+ module LinkThumbnailer
4
+
5
+ class DocParser
6
+
7
+ def parse(doc_string, source_url = nil)
8
+ doc = Nokogiri::HTML(doc_string).extend(LinkThumbnailer::Doc)
9
+ doc.source_url = source_url
10
+ doc
11
+ end
12
+
13
+ end
14
+
15
+ end
@@ -1,4 +1,4 @@
1
- module LinkThumbnailer
2
- class Engine < ::Rails::Engine
3
- end
4
- end
1
+ module LinkThumbnailer
2
+ class Engine < ::Rails::Engine
3
+ end
4
+ end
@@ -1,34 +1,34 @@
1
- require 'net/http/persistent'
2
-
3
- module LinkThumbnailer
4
-
5
- class Fetcher
6
-
7
- attr_accessor :url
8
-
9
- def fetch(url, redirect_count = 0)
10
- if redirect_count > LinkThumbnailer.configuration.redirect_limit
11
- raise ArgumentError, "too many redirects (#{redirect_count})"
12
- end
13
-
14
- self.url = url.is_a?(URI) ? url : URI(url)
15
-
16
- if self.url.is_a?(URI::HTTP)
17
- http = Net::HTTP::Persistent.new('linkthumbnailer')
18
- http.headers['User-Agent'] = LinkThumbnailer.configuration.user_agent
19
- http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless LinkThumbnailer.configuration.verify_ssl
20
- http.open_timeout = LinkThumbnailer.configuration.http_timeout
21
- resp = http.request(self.url)
22
- case resp
23
- when Net::HTTPSuccess then resp.body
24
- when Net::HTTPRedirection
25
- location = resp['location'].start_with?('http') ? resp['location'] : "#{self.url.scheme}://#{self.url.host}#{resp['location']}"
26
- fetch(location, redirect_count + 1)
27
- else resp.error!
28
- end
29
- end
30
- end
31
-
32
- end
33
-
34
- end
1
+ require 'net/http/persistent'
2
+
3
+ module LinkThumbnailer
4
+
5
+ class Fetcher
6
+
7
+ attr_accessor :url
8
+
9
+ def fetch(url, redirect_count = 0)
10
+ if redirect_count > LinkThumbnailer.configuration.redirect_limit
11
+ raise ArgumentError, "too many redirects (#{redirect_count})"
12
+ end
13
+
14
+ self.url = url.is_a?(URI) ? url : URI(url)
15
+
16
+ if self.url.is_a?(URI::HTTP)
17
+ http = Net::HTTP::Persistent.new('linkthumbnailer')
18
+ http.headers['User-Agent'] = LinkThumbnailer.configuration.user_agent
19
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless LinkThumbnailer.configuration.verify_ssl
20
+ http.open_timeout = LinkThumbnailer.configuration.http_timeout
21
+ resp = http.request(self.url)
22
+ case resp
23
+ when Net::HTTPSuccess then resp.body
24
+ when Net::HTTPRedirection
25
+ location = resp['location'].start_with?('http') ? resp['location'] : "#{self.url.scheme}://#{self.url.host}#{resp['location']}"
26
+ fetch(location, redirect_count + 1)
27
+ else resp.error!
28
+ end
29
+ end
30
+ end
31
+
32
+ end
33
+
34
+ end
@@ -1,17 +1,17 @@
1
- module LinkThumbnailer
2
-
3
- module ImgComparator
4
-
5
- def <=> other
6
- if other.size != nil && size != nil
7
- return (other.size.min ** 2) <=> (size.min ** 2)
8
- elsif other.size != nil
9
- return 1
10
- else
11
- return -1
12
- end
13
- end
14
-
15
- end
16
-
17
- end
1
+ module LinkThumbnailer
2
+
3
+ module ImgComparator
4
+
5
+ def <=> other
6
+ if other.size != nil && size != nil
7
+ return (other.size.min ** 2) <=> (size.min ** 2)
8
+ elsif other.size != nil
9
+ return 1
10
+ else
11
+ return -1
12
+ end
13
+ end
14
+
15
+ end
16
+
17
+ end
@@ -1,41 +1,41 @@
1
- require 'fastimage'
2
-
3
- module LinkThumbnailer
4
-
5
- class ImgParser
6
-
7
- def initialize(fetcher, img_url_filter)
8
- @fetcher = fetcher
9
- @img_url_filters = [*img_url_filter]
10
- end
11
-
12
- def parse(img_urls)
13
- @img_url_filters.each do |filter|
14
- img_urls.delete_if { |i| filter.reject?(i) }
15
- end
16
-
17
- imgs = []
18
- count = 0
19
- img_urls.each { |i|
20
- break if count >= LinkThumbnailer.configuration.limit
21
- img = parse_one(i)
22
- next unless img
23
- img.extend LinkThumbnailer::WebImage
24
- img.extend LinkThumbnailer::ImgComparator
25
- imgs << img
26
- count += 1
27
- }
28
-
29
- imgs.sort! unless imgs.count <= 1
30
- imgs.first(LinkThumbnailer.configuration.top)
31
- end
32
-
33
- def parse_one(img_url)
34
- FastImage.new(img_url, raise_on_failure: false)
35
- rescue StandardError
36
- nil
37
- end
38
-
39
- end
40
-
41
- end
1
+ require 'fastimage'
2
+
3
+ module LinkThumbnailer
4
+
5
+ class ImgParser
6
+
7
+ def initialize(fetcher, img_url_filter)
8
+ @fetcher = fetcher
9
+ @img_url_filters = [*img_url_filter]
10
+ end
11
+
12
+ def parse(img_urls)
13
+ @img_url_filters.each do |filter|
14
+ img_urls.delete_if { |i| filter.reject?(i) }
15
+ end
16
+
17
+ imgs = []
18
+ count = 0
19
+ img_urls.each { |i|
20
+ break if count >= LinkThumbnailer.configuration.limit
21
+ img = parse_one(i)
22
+ next unless img
23
+ img.extend LinkThumbnailer::WebImage
24
+ img.extend LinkThumbnailer::ImgComparator
25
+ imgs << img
26
+ count += 1
27
+ }
28
+
29
+ imgs.sort! unless imgs.count <= 1
30
+ imgs.first(LinkThumbnailer.configuration.top)
31
+ end
32
+
33
+ def parse_one(img_url)
34
+ FastImage.new(img_url.to_s, raise_on_failure: false)
35
+ rescue StandardError
36
+ nil
37
+ end
38
+
39
+ end
40
+
41
+ end
@@ -1,13 +1,13 @@
1
- module LinkThumbnailer
2
- class ImgUrlFilter
3
-
4
- def reject?(img_url)
5
- LinkThumbnailer.configuration.blacklist_urls.each do |url|
6
- return true if img_url && img_url.to_s[url]
7
- end
8
-
9
- false
10
- end
11
-
12
- end
13
- end
1
+ module LinkThumbnailer
2
+ class ImgUrlFilter
3
+
4
+ def reject?(img_url)
5
+ LinkThumbnailer.configuration.blacklist_urls.each do |url|
6
+ return true if img_url && img_url.to_s[url]
7
+ end
8
+
9
+ false
10
+ end
11
+
12
+ end
13
+ end
@@ -1,41 +1,41 @@
1
- require 'hashie'
2
- require 'json'
3
-
4
- module LinkThumbnailer
5
- class Object < Hashie::Mash
6
-
7
- def method_missing(method_name, *args, &block)
8
- method_name = method_name.to_s
9
-
10
- if method_name.end_with?('?')
11
- method_name.chop!
12
- !self[method_name].nil?
13
- else
14
- self[method_name]
15
- end
16
- end
17
-
18
- def valid?
19
- return false if keys.empty?
20
- LinkThumbnailer.configuration.mandatory_attributes.each { |a| return false if self[a].nil? || self[a].empty? } if LinkThumbnailer.configuration.strict
21
- true
22
- end
23
-
24
- def to_hash
25
- if images.none? { |i| i.is_a?(String) }
26
- super.merge('images' => images.map(&:to_hash))
27
- else
28
- super
29
- end
30
- end
31
-
32
- def to_json
33
- if images.none? { |i| i.is_a?(String) }
34
- JSON.generate(to_hash.merge('images' => images.map(&:to_hash)))
35
- else
36
- JSON.generate(to_hash)
37
- end
38
- end
39
-
40
- end
41
- end
1
+ require 'hashie'
2
+ require 'json'
3
+
4
+ module LinkThumbnailer
5
+ class Object < Hashie::Mash
6
+
7
+ def method_missing(method_name, *args, &block)
8
+ method_name = method_name.to_s
9
+
10
+ if method_name.end_with?('?')
11
+ method_name.chop!
12
+ !self[method_name].nil?
13
+ else
14
+ self[method_name]
15
+ end
16
+ end
17
+
18
+ def valid?
19
+ return false if keys.empty?
20
+ LinkThumbnailer.configuration.mandatory_attributes.each { |a| return false if self[a].nil? || self[a].empty? } if LinkThumbnailer.configuration.strict
21
+ true
22
+ end
23
+
24
+ def to_hash
25
+ if images.none? { |i| i.is_a?(String) }
26
+ super.merge('images' => images.map(&:to_hash))
27
+ else
28
+ super
29
+ end
30
+ end
31
+
32
+ def to_json
33
+ if images.none? { |i| i.is_a?(String) }
34
+ JSON.generate(to_hash.merge('images' => images.map(&:to_hash)))
35
+ else
36
+ JSON.generate(to_hash)
37
+ end
38
+ end
39
+
40
+ end
41
+ end
@@ -1,20 +1,20 @@
1
- module LinkThumbnailer
2
- class Opengraph
3
-
4
- def self.parse(object, doc)
5
- doc.css('meta').each do |m|
6
- if m.attribute('property') && m.attribute('property').to_s.match(/^og:(.+)$/i)
7
- object[$1.gsub('-', '_')] = m.attribute('content').to_s
8
- end
9
- end
10
-
11
- object[:images] = []
12
- if object[:image]
13
- object[:images] << { source_url: object[:image] }
14
- end
15
-
16
- object
17
- end
18
-
19
- end
20
- end
1
+ module LinkThumbnailer
2
+ class Opengraph
3
+
4
+ def self.parse(object, doc)
5
+ doc.css('meta').each do |m|
6
+ if m.attribute('property') && m.attribute('property').to_s.match(/^og:(.+)$/i)
7
+ object[$1.gsub('-', '_')] = m.attribute('content').to_s
8
+ end
9
+ end
10
+
11
+ object[:images] = []
12
+ if object[:image]
13
+ object[:images] << { source_url: object[:image] }
14
+ end
15
+
16
+ object
17
+ end
18
+
19
+ end
20
+ end
@@ -1,9 +1,9 @@
1
- module LinkThumbnailer
2
- class Railtie < ::Rails::Railtie
3
-
4
- initializer 'link_thumbnailer.routes' do
5
- LinkThumbnailer::Rails::Routes.install!
6
- end
7
-
8
- end
9
- end
1
+ module LinkThumbnailer
2
+ class Railtie < ::Rails::Railtie
3
+
4
+ initializer 'link_thumbnailer.routes' do
5
+ LinkThumbnailer::Rails::Routes.install!
6
+ end
7
+
8
+ end
9
+ end
@@ -1,3 +1,3 @@
1
- module LinkThumbnailer
2
- VERSION = "1.1.1"
3
- end
1
+ module LinkThumbnailer
2
+ VERSION = "1.1.2"
3
+ end