link_thumbnailer 1.1.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. checksums.yaml +5 -13
  2. data/.travis.yml +1 -1
  3. data/CHANGELOG.md +117 -104
  4. data/Gemfile +1 -1
  5. data/{LICENSE → LICENSE.txt} +21 -21
  6. data/README.md +153 -184
  7. data/lib/generators/link_thumbnailer/install_generator.rb +0 -4
  8. data/lib/generators/templates/initializer.rb +63 -41
  9. data/lib/link_thumbnailer/configuration.rb +52 -10
  10. data/lib/link_thumbnailer/exceptions.rb +6 -0
  11. data/lib/link_thumbnailer/grader.rb +37 -0
  12. data/lib/link_thumbnailer/graders/base.rb +32 -0
  13. data/lib/link_thumbnailer/graders/html_attribute.rb +49 -0
  14. data/lib/link_thumbnailer/graders/length.rb +19 -0
  15. data/lib/link_thumbnailer/graders/link_density.rb +21 -0
  16. data/lib/link_thumbnailer/graders/position.rb +11 -0
  17. data/lib/link_thumbnailer/image_comparator.rb +24 -0
  18. data/lib/link_thumbnailer/image_comparators/base.rb +17 -0
  19. data/lib/link_thumbnailer/image_comparators/size.rb +11 -0
  20. data/lib/link_thumbnailer/image_parser.rb +18 -0
  21. data/lib/link_thumbnailer/image_parsers/size.rb +15 -0
  22. data/lib/link_thumbnailer/image_parsers/type.rb +15 -0
  23. data/lib/link_thumbnailer/image_validator.rb +30 -0
  24. data/lib/link_thumbnailer/model.rb +16 -0
  25. data/lib/link_thumbnailer/models/description.rb +34 -0
  26. data/lib/link_thumbnailer/models/image.rb +54 -0
  27. data/lib/link_thumbnailer/models/title.rb +20 -0
  28. data/lib/link_thumbnailer/models/website.rb +39 -0
  29. data/lib/link_thumbnailer/page.rb +40 -0
  30. data/lib/link_thumbnailer/parser.rb +13 -0
  31. data/lib/link_thumbnailer/processor.rb +94 -0
  32. data/lib/link_thumbnailer/railtie.rb +9 -9
  33. data/lib/link_thumbnailer/scraper.rb +64 -0
  34. data/lib/link_thumbnailer/scrapers/base.rb +63 -0
  35. data/lib/link_thumbnailer/scrapers/default/base.rb +10 -0
  36. data/lib/link_thumbnailer/scrapers/default/description.rb +47 -0
  37. data/lib/link_thumbnailer/scrapers/default/images.rb +64 -0
  38. data/lib/link_thumbnailer/scrapers/default/title.rb +25 -0
  39. data/lib/link_thumbnailer/scrapers/opengraph/base.rb +43 -0
  40. data/lib/link_thumbnailer/scrapers/opengraph/description.rb +10 -0
  41. data/lib/link_thumbnailer/scrapers/opengraph/image.rb +30 -0
  42. data/lib/link_thumbnailer/scrapers/opengraph/images.rb +16 -0
  43. data/lib/link_thumbnailer/scrapers/opengraph/title.rb +10 -0
  44. data/lib/link_thumbnailer/version.rb +3 -3
  45. data/lib/link_thumbnailer.rb +36 -119
  46. data/link_thumbnailer.gemspec +26 -28
  47. data/spec/configuration_spec.rb +51 -0
  48. data/spec/examples/empty_og_image_example.html +9 -0
  49. data/spec/fixture_spec.rb +88 -0
  50. data/spec/fixtures/bar.png +2907 -0
  51. data/spec/fixtures/default_from_body.html +12 -0
  52. data/spec/fixtures/default_from_meta.html +11 -0
  53. data/spec/{examples → fixtures}/example.html +53 -53
  54. data/spec/fixtures/foo.png +0 -0
  55. data/spec/fixtures/og_not_valid_example.html +12 -0
  56. data/spec/fixtures/og_valid_example.html +12 -0
  57. data/spec/fixtures/og_valid_multi_image_example.html +13 -0
  58. data/spec/grader_spec.rb +24 -0
  59. data/spec/graders/base_spec.rb +12 -0
  60. data/spec/graders/html_attribute_spec.rb +48 -0
  61. data/spec/graders/length_spec.rb +81 -0
  62. data/spec/graders/link_density_spec.rb +22 -0
  63. data/spec/image_comparators/size_spec.rb +39 -0
  64. data/spec/image_parsers/size_spec.rb +34 -0
  65. data/spec/image_parsers/type_spec.rb +34 -0
  66. data/spec/image_validator_spec.rb +35 -0
  67. data/spec/model_spec.rb +17 -0
  68. data/spec/models/description_spec.rb +64 -0
  69. data/spec/models/image_spec.rb +71 -0
  70. data/spec/models/title_spec.rb +24 -0
  71. data/spec/models/website_spec.rb +49 -0
  72. data/spec/page_spec.rb +26 -0
  73. data/spec/processor_spec.rb +349 -0
  74. data/spec/scraper_spec.rb +95 -0
  75. data/spec/scrapers/base_spec.rb +67 -0
  76. data/spec/scrapers/opengraph/base_spec.rb +94 -0
  77. data/spec/spec_helper.rb +15 -13
  78. metadata +126 -120
  79. data/app/controllers/link_thumbnailer/application_controller.rb +0 -4
  80. data/app/controllers/link_thumbnailer/previews_controller.rb +0 -11
  81. data/lib/link_thumbnailer/doc.rb +0 -65
  82. data/lib/link_thumbnailer/doc_parser.rb +0 -15
  83. data/lib/link_thumbnailer/engine.rb +0 -4
  84. data/lib/link_thumbnailer/fetcher.rb +0 -34
  85. data/lib/link_thumbnailer/img_comparator.rb +0 -17
  86. data/lib/link_thumbnailer/img_parser.rb +0 -41
  87. data/lib/link_thumbnailer/img_url_filter.rb +0 -13
  88. data/lib/link_thumbnailer/object.rb +0 -41
  89. data/lib/link_thumbnailer/opengraph.rb +0 -20
  90. data/lib/link_thumbnailer/rails/routes/mapper.rb +0 -30
  91. data/lib/link_thumbnailer/rails/routes/mapping.rb +0 -33
  92. data/lib/link_thumbnailer/rails/routes.rb +0 -47
  93. data/lib/link_thumbnailer/web_image.rb +0 -19
  94. data/spec/doc_parser_spec.rb +0 -25
  95. data/spec/doc_spec.rb +0 -23
  96. data/spec/examples/empty_example.html +0 -11
  97. data/spec/examples/og_example.html +0 -12
  98. data/spec/fetcher_spec.rb +0 -97
  99. data/spec/img_comparator_spec.rb +0 -16
  100. data/spec/img_url_filter_spec.rb +0 -31
  101. data/spec/link_thumbnailer_spec.rb +0 -205
  102. data/spec/object_spec.rb +0 -130
  103. data/spec/opengraph_spec.rb +0 -7
  104. data/spec/web_image_spec.rb +0 -57
@@ -1,17 +0,0 @@
1
- module LinkThumbnailer
2
-
3
- module ImgComparator
4
-
5
- def <=> other
6
- if other.size != nil && size != nil
7
- return (other.size.min ** 2) <=> (size.min ** 2)
8
- elsif other.size != nil
9
- return 1
10
- else
11
- return -1
12
- end
13
- end
14
-
15
- end
16
-
17
- end
@@ -1,41 +0,0 @@
1
- require 'fastimage'
2
-
3
- module LinkThumbnailer
4
-
5
- class ImgParser
6
-
7
- def initialize(fetcher, img_url_filter)
8
- @fetcher = fetcher
9
- @img_url_filters = [*img_url_filter]
10
- end
11
-
12
- def parse(img_urls)
13
- @img_url_filters.each do |filter|
14
- img_urls.delete_if { |i| filter.reject?(i) }
15
- end
16
-
17
- imgs = []
18
- count = 0
19
- img_urls.each { |i|
20
- break if count >= LinkThumbnailer.configuration.limit
21
- img = parse_one(i)
22
- next unless img
23
- img.extend LinkThumbnailer::WebImage
24
- img.extend LinkThumbnailer::ImgComparator
25
- imgs << img
26
- count += 1
27
- }
28
-
29
- imgs.sort! unless imgs.count <= 1
30
- imgs.first(LinkThumbnailer.configuration.top)
31
- end
32
-
33
- def parse_one(img_url)
34
- FastImage.new(img_url.to_s, raise_on_failure: false)
35
- rescue StandardError
36
- nil
37
- end
38
-
39
- end
40
-
41
- end
@@ -1,13 +0,0 @@
1
- module LinkThumbnailer
2
- class ImgUrlFilter
3
-
4
- def reject?(img_url)
5
- LinkThumbnailer.configuration.blacklist_urls.each do |url|
6
- return true if img_url && img_url.to_s[url]
7
- end
8
-
9
- false
10
- end
11
-
12
- end
13
- end
@@ -1,41 +0,0 @@
1
- require 'hashie'
2
- require 'json'
3
-
4
- module LinkThumbnailer
5
- class Object < Hashie::Mash
6
-
7
- def method_missing(method_name, *args, &block)
8
- method_name = method_name.to_s
9
-
10
- if method_name.end_with?('?')
11
- method_name.chop!
12
- !self[method_name].nil?
13
- else
14
- self[method_name]
15
- end
16
- end
17
-
18
- def valid?
19
- return false if keys.empty?
20
- LinkThumbnailer.configuration.mandatory_attributes.each { |a| return false if self[a].nil? || self[a].empty? } if LinkThumbnailer.configuration.strict
21
- true
22
- end
23
-
24
- def to_hash
25
- if images.none? { |i| i.is_a?(String) }
26
- super.merge('images' => images.map(&:to_hash))
27
- else
28
- super
29
- end
30
- end
31
-
32
- def to_json
33
- if images.none? { |i| i.is_a?(String) }
34
- JSON.generate(to_hash.merge('images' => images.map(&:to_hash)))
35
- else
36
- JSON.generate(to_hash)
37
- end
38
- end
39
-
40
- end
41
- end
@@ -1,20 +0,0 @@
1
- module LinkThumbnailer
2
- class Opengraph
3
-
4
- def self.parse(object, doc)
5
- doc.css('meta').each do |m|
6
- if m.attribute('property') && m.attribute('property').to_s.match(/^og:(.+)$/i)
7
- object[$1.gsub('-', '_')] = m.attribute('content').to_s
8
- end
9
- end
10
-
11
- object[:images] = []
12
- if object[:image]
13
- object[:images] << { source_url: object[:image] }
14
- end
15
-
16
- object
17
- end
18
-
19
- end
20
- end
@@ -1,30 +0,0 @@
1
- module LinkThumbnailer
2
- module Rails
3
- class Routes
4
- class Mapper
5
-
6
- def initialize(mapping = Mapping.new)
7
- @mapping = mapping
8
- end
9
-
10
- def map(&block)
11
- self.instance_eval(&block) if block
12
- @mapping
13
- end
14
-
15
- def controllers(controller_names = {})
16
- @mapping.controllers.merge!(controller_names)
17
- end
18
-
19
- def skip_controllers(*controller_names)
20
- @mapping.skips = controller_names
21
- end
22
-
23
- def as(alias_names = {})
24
- @mapping.as.merge!(alias_names)
25
- end
26
-
27
- end
28
- end
29
- end
30
- end
@@ -1,33 +0,0 @@
1
- module LinkThumbnailer
2
- module Rails
3
- class Routes
4
- class Mapping
5
-
6
- attr_accessor :controllers, :as, :skips
7
-
8
- def initialize
9
- @controllers = {
10
- previews: 'link_thumbnailer/previews'
11
- }
12
-
13
- @as = {
14
- previews: :preview
15
- }
16
-
17
- @skips = []
18
- end
19
-
20
- def [](routes)
21
- {
22
- controllers: @controllers[routes],
23
- as: @as[routes]
24
- }
25
- end
26
-
27
- def skipped?(controller)
28
- @skips.include?(controller)
29
- end
30
- end
31
- end
32
- end
33
- end
@@ -1,47 +0,0 @@
1
- require 'link_thumbnailer/rails/routes/mapping'
2
- require 'link_thumbnailer/rails/routes/mapper'
3
-
4
- module LinkThumbnailer
5
- module Rails
6
- class Routes
7
-
8
- module Helper
9
- def use_link_thumbnailer(options = {}, &block)
10
- LinkThumbnailer::Rails::Routes.new(self, &block).generate_routes!(options)
11
- end
12
- end
13
-
14
- def self.install!
15
- ActionDispatch::Routing::Mapper.send(:include, LinkThumbnailer::Rails::Routes::Helper)
16
- end
17
-
18
- attr_accessor :routes
19
-
20
- def initialize(routes, &options)
21
- @routes, @options = routes, options
22
- end
23
-
24
- def generate_routes!(options)
25
- @mapping = Mapper.new.map(&@options)
26
- routes.scope 'link', as: 'link' do
27
- map_route(:previews, :preview_routes)
28
- end
29
- end
30
-
31
- private
32
-
33
- def map_route(name, method)
34
- unless @mapping.skipped?(name)
35
- send method, @mapping[name]
36
- end
37
- end
38
-
39
- def preview_routes(mapping)
40
- routes.scope controller: mapping[:controllers] do
41
- routes.match 'preview', via: :post, action: :create, as: mapping[:as], defaults: { format: 'json' }
42
- end
43
- end
44
-
45
- end
46
- end
47
- end
@@ -1,19 +0,0 @@
1
- module LinkThumbnailer
2
- module WebImage
3
-
4
- attr_accessor :doc
5
-
6
- def source_url
7
- instance_variable_get(:@uri)
8
- end
9
-
10
- def to_hash
11
- LinkThumbnailer.configuration.image_attributes.each_with_object({}) { |m, result|
12
- k = m.to_sym
13
- result[k] = self.send(m) if self.respond_to?(m)
14
- result[k] = result[k].to_s if result[k].is_a?(URI) || result[k].respond_to?(:to_str)
15
- }
16
- end
17
-
18
- end
19
- end
@@ -1,25 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe LinkThumbnailer::DocParser do
4
-
5
- it { should respond_to(:parse).with(1).arguments }
6
-
7
- let(:instance) { LinkThumbnailer::DocParser.new }
8
-
9
- describe "#parse" do
10
-
11
- let(:source_url) { 'http://foo.com' }
12
-
13
- subject { instance.parse('', source_url) }
14
-
15
- it { expect(subject.source_url).to eq(source_url) }
16
- it { expect(subject).to respond_to(:doc_base_href) }
17
- it { expect(subject).to respond_to(:img_srcs) }
18
- it { expect(subject).to respond_to(:img_abs_urls) }
19
- it { expect(subject).to respond_to(:title) }
20
- it { expect(subject).to respond_to(:description) }
21
- it { expect(subject).to respond_to(:source_url) }
22
-
23
- end
24
-
25
- end
data/spec/doc_spec.rb DELETED
@@ -1,23 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe LinkThumbnailer::Doc do
4
-
5
- class Foo
6
- end
7
-
8
- let(:foo) { Foo.new }
9
-
10
- before do
11
- foo.extend LinkThumbnailer::Doc
12
- end
13
-
14
- subject { foo }
15
-
16
- it { should respond_to :doc_base_href }
17
- it { should respond_to :img_srcs }
18
- it { should respond_to :img_abs_urls }
19
- it { should respond_to :title }
20
- it { should respond_to :description }
21
- it { should respond_to :source_url }
22
-
23
- end
@@ -1,11 +0,0 @@
1
- <!DOCTYPE html>
2
- <html>
3
- <head>
4
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
5
- <meta charset="UTF-8">
6
- <title>Foo.com</title>
7
- </head>
8
-
9
- <body>
10
- </body>
11
- </html>
@@ -1,12 +0,0 @@
1
- <html xmlns:og="http://opengraphprotocol.org/schema/">
2
- <head>
3
- <meta charset="utf-8"/>
4
- <meta property="og:type" content="website"/>
5
- <meta property="og:site_name" content="foo.com">
6
- <meta property="og:title" content="Foo Title">
7
- <meta property="og:description" content="Foo description">
8
- <meta property="og:image" content="http://foo.com/img/front/facebook.png">
9
- <title>Foo</title>
10
- </head>
11
-
12
- </html>
data/spec/fetcher_spec.rb DELETED
@@ -1,97 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe LinkThumbnailer::Fetcher do
4
-
5
- it { should respond_to :fetch }
6
- it { should respond_to :url }
7
- it { should respond_to :url= }
8
-
9
- let(:fetcher) { LinkThumbnailer::Fetcher.new }
10
- let(:url) { 'http://foo.com' }
11
-
12
- describe ".fetch" do
13
-
14
- before do
15
- LinkThumbnailer.configure {|config| config.redirect_count = 3}
16
- end
17
-
18
- context "when redirect_count is more than config" do
19
-
20
- it { expect { fetcher.fetch(url, 10) }.to raise_exception(ArgumentError) }
21
-
22
- end
23
-
24
- context "when no http error" do
25
-
26
- before do
27
- stub_request(:get, url).to_return(status: 200, body: 'foo', headers: {})
28
- end
29
-
30
- it "returns body response" do
31
- fetcher.fetch(url).should eq('foo')
32
- end
33
-
34
- it "sets fetcher url" do
35
- fetcher.fetch(url)
36
- fetcher.url.to_s.should eq(url)
37
- end
38
-
39
- end
40
-
41
- context "when http redirection" do
42
-
43
- context "and relative uri" do
44
-
45
- let(:another_url) { '/bar' }
46
-
47
- before do
48
- stub_request(:get, url).to_return(status: 300, body: 'foo', headers: { 'Location' => another_url })
49
- stub_request(:get, url + another_url).to_return(status: 200, body: 'bar', headers: {})
50
- end
51
-
52
- it "returns body response" do
53
- fetcher.fetch(url).should eq('bar')
54
- end
55
-
56
- it "sets fetcher url" do
57
- fetcher.fetch(url)
58
- fetcher.url.to_s.should eq(url + another_url)
59
- end
60
-
61
- end
62
-
63
- context "and absolute uri" do
64
-
65
- let(:another_url) { 'http://bar.com' }
66
-
67
- before do
68
- stub_request(:get, url).to_return(status: 300, body: 'foo', headers: { 'Location' => another_url })
69
- stub_request(:get, another_url).to_return(status: 200, body: 'bar', headers: {})
70
- end
71
-
72
- it "returns body response" do
73
- fetcher.fetch(url).should eq('bar')
74
- end
75
-
76
- it "sets fetcher url" do
77
- fetcher.fetch(url)
78
- fetcher.url.to_s.should eq(another_url)
79
- end
80
-
81
- end
82
-
83
- end
84
-
85
- context "when http error" do
86
-
87
- before do
88
- stub_request(:get, url).to_return(status: 500, body: 'foo', headers: {})
89
- end
90
-
91
- it { expect { fetcher.fetch(url) }.to raise_exception(Net::HTTPFatalError) }
92
-
93
- end
94
-
95
- end
96
-
97
- end
@@ -1,16 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe LinkThumbnailer::ImgComparator do
4
-
5
- class Foo
6
- end
7
-
8
- let(:foo) { Foo.new }
9
-
10
- before do
11
- foo.extend(LinkThumbnailer::ImgComparator)
12
- end
13
-
14
- it { should respond_to :<=> }
15
-
16
- end
@@ -1,31 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe LinkThumbnailer::ImgUrlFilter do
4
-
5
- it { should respond_to :reject? }
6
-
7
- describe ".reject?" do
8
-
9
- let(:img_url_filter) { LinkThumbnailer::ImgUrlFilter.new }
10
-
11
- before do
12
- LinkThumbnailer.configure {|config| config.blacklist_urls = [
13
- %r{^http://not_valid\.net/}
14
- ]}
15
- end
16
-
17
- context "when img_url does not contain any blacklisted urls" do
18
-
19
- it { expect(img_url_filter.reject?('http://valid.com/foo/bar.png')).to be_false }
20
-
21
- end
22
-
23
- context "when img_url does contain any blacklisted urls" do
24
-
25
- it { expect(img_url_filter.reject?('http://not_valid.net/foo/bar.png')).to be_true }
26
-
27
- end
28
-
29
- end
30
-
31
- end