link_thumbnailer 3.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +19 -0
  3. data/.rspec +2 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +5 -0
  6. data/CHANGELOG.md +334 -0
  7. data/Gemfile +12 -0
  8. data/LICENSE.txt +22 -0
  9. data/README.md +210 -0
  10. data/Rakefile +9 -0
  11. data/lib/generators/link_thumbnailer/install_generator.rb +17 -0
  12. data/lib/generators/templates/initializer.rb +89 -0
  13. data/lib/link_thumbnailer.rb +38 -0
  14. data/lib/link_thumbnailer/configuration.rb +72 -0
  15. data/lib/link_thumbnailer/exceptions.rb +11 -0
  16. data/lib/link_thumbnailer/grader.rb +43 -0
  17. data/lib/link_thumbnailer/graders/base.rb +39 -0
  18. data/lib/link_thumbnailer/graders/html_attribute.rb +48 -0
  19. data/lib/link_thumbnailer/graders/length.rb +37 -0
  20. data/lib/link_thumbnailer/graders/link_density.rb +20 -0
  21. data/lib/link_thumbnailer/graders/position.rb +13 -0
  22. data/lib/link_thumbnailer/image_comparator.rb +26 -0
  23. data/lib/link_thumbnailer/image_comparators/base.rb +19 -0
  24. data/lib/link_thumbnailer/image_comparators/size.rb +13 -0
  25. data/lib/link_thumbnailer/image_parser.rb +62 -0
  26. data/lib/link_thumbnailer/image_validator.rb +32 -0
  27. data/lib/link_thumbnailer/model.rb +20 -0
  28. data/lib/link_thumbnailer/models/description.rb +37 -0
  29. data/lib/link_thumbnailer/models/favicon.rb +27 -0
  30. data/lib/link_thumbnailer/models/image.rb +56 -0
  31. data/lib/link_thumbnailer/models/title.rb +22 -0
  32. data/lib/link_thumbnailer/models/video.rb +44 -0
  33. data/lib/link_thumbnailer/models/website.rb +54 -0
  34. data/lib/link_thumbnailer/page.rb +43 -0
  35. data/lib/link_thumbnailer/parser.rb +15 -0
  36. data/lib/link_thumbnailer/processor.rb +128 -0
  37. data/lib/link_thumbnailer/railtie.rb +6 -0
  38. data/lib/link_thumbnailer/response.rb +39 -0
  39. data/lib/link_thumbnailer/scraper.rb +62 -0
  40. data/lib/link_thumbnailer/scrapers/base.rb +69 -0
  41. data/lib/link_thumbnailer/scrapers/default/base.rb +12 -0
  42. data/lib/link_thumbnailer/scrapers/default/description.rb +49 -0
  43. data/lib/link_thumbnailer/scrapers/default/favicon.rb +38 -0
  44. data/lib/link_thumbnailer/scrapers/default/images.rb +78 -0
  45. data/lib/link_thumbnailer/scrapers/default/title.rb +27 -0
  46. data/lib/link_thumbnailer/scrapers/default/videos.rb +18 -0
  47. data/lib/link_thumbnailer/scrapers/opengraph/base.rb +45 -0
  48. data/lib/link_thumbnailer/scrapers/opengraph/description.rb +12 -0
  49. data/lib/link_thumbnailer/scrapers/opengraph/favicon.rb +17 -0
  50. data/lib/link_thumbnailer/scrapers/opengraph/image.rb +107 -0
  51. data/lib/link_thumbnailer/scrapers/opengraph/images.rb +18 -0
  52. data/lib/link_thumbnailer/scrapers/opengraph/title.rb +12 -0
  53. data/lib/link_thumbnailer/scrapers/opengraph/video.rb +115 -0
  54. data/lib/link_thumbnailer/scrapers/opengraph/videos.rb +18 -0
  55. data/lib/link_thumbnailer/uri.rb +20 -0
  56. data/lib/link_thumbnailer/version.rb +5 -0
  57. data/lib/link_thumbnailer/video_parser.rb +47 -0
  58. data/link_thumbnailer.gemspec +29 -0
  59. data/spec/configuration_spec.rb +61 -0
  60. data/spec/fixture_spec.rb +114 -0
  61. data/spec/fixtures/bar.png +2907 -0
  62. data/spec/fixtures/default_from_body.html +13 -0
  63. data/spec/fixtures/default_from_meta.html +12 -0
  64. data/spec/fixtures/foo.png +0 -0
  65. data/spec/fixtures/google_shift_jis.html +6 -0
  66. data/spec/fixtures/google_utf8.html +6 -0
  67. data/spec/fixtures/og_not_valid_example.html +12 -0
  68. data/spec/fixtures/og_valid_example.html +18 -0
  69. data/spec/fixtures/og_valid_multi_image_example.html +13 -0
  70. data/spec/fixtures/og_valid_multi_video_example.html +13 -0
  71. data/spec/grader_spec.rb +27 -0
  72. data/spec/graders/base_spec.rb +14 -0
  73. data/spec/graders/html_attribute_spec.rb +50 -0
  74. data/spec/graders/length_spec.rb +93 -0
  75. data/spec/graders/link_density_spec.rb +52 -0
  76. data/spec/graders/position_spec.rb +49 -0
  77. data/spec/image_comparators/size_spec.rb +58 -0
  78. data/spec/image_validator_spec.rb +37 -0
  79. data/spec/model_spec.rb +27 -0
  80. data/spec/models/description_spec.rb +66 -0
  81. data/spec/models/favicon_spec.rb +12 -0
  82. data/spec/models/image_spec.rb +95 -0
  83. data/spec/models/title_spec.rb +26 -0
  84. data/spec/models/video_spec.rb +49 -0
  85. data/spec/models/website_spec.rb +51 -0
  86. data/spec/page_spec.rb +28 -0
  87. data/spec/processor_spec.rb +410 -0
  88. data/spec/response_spec.rb +62 -0
  89. data/spec/scraper_spec.rb +70 -0
  90. data/spec/scrapers/base_spec.rb +69 -0
  91. data/spec/scrapers/opengraph/base_spec.rb +96 -0
  92. data/spec/spec_helper.rb +11 -0
  93. data/spec/uri_spec.rb +44 -0
  94. data/spec/video_parser_spec.rb +148 -0
  95. metadata +271 -0
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'link_thumbnailer/scrapers/default/base'
4
+
5
+ module LinkThumbnailer
6
+ module Scrapers
7
+ module Default
8
+ class Title < ::LinkThumbnailer::Scrapers::Default::Base
9
+
10
+ def value
11
+ model.to_s
12
+ end
13
+
14
+ private
15
+
16
+ def model
17
+ modelize(node)
18
+ end
19
+
20
+ def node
21
+ document.css(attribute_name)
22
+ end
23
+
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'link_thumbnailer/scrapers/default/base'
4
+ require 'link_thumbnailer/models/video'
5
+
6
+ module LinkThumbnailer
7
+ module Scrapers
8
+ module Default
9
+ class Videos < ::LinkThumbnailer::Scrapers::Default::Base
10
+
11
+ def value
12
+ nil
13
+ end
14
+
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'link_thumbnailer/scrapers/base'
4
+
5
+ module LinkThumbnailer
6
+ module Scrapers
7
+ module Opengraph
8
+ class Base < ::LinkThumbnailer::Scrapers::Base
9
+
10
+ def applicable?
11
+ meta.any? { |node| opengraph_node?(node) }
12
+ end
13
+
14
+ def value
15
+ model.to_s
16
+ end
17
+
18
+ private
19
+
20
+ def model
21
+ modelize(node, node.attributes['content'].to_s) if node
22
+ end
23
+
24
+ def node
25
+ @node ||= meta_xpath(attribute: attribute) ||
26
+ meta_xpath(attribute: attribute, key: :name)
27
+ end
28
+
29
+ def attribute
30
+ "og:#{attribute_name}"
31
+ end
32
+
33
+ def opengraph_node?(node)
34
+ node.attribute('name').to_s.start_with?('og:') ||
35
+ node.attribute('property').to_s.start_with?('og:')
36
+ end
37
+
38
+ def meta
39
+ document.css('meta')
40
+ end
41
+
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'link_thumbnailer/scrapers/opengraph/base'
4
+
5
+ module LinkThumbnailer
6
+ module Scrapers
7
+ module Opengraph
8
+ class Description < ::LinkThumbnailer::Scrapers::Opengraph::Base
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'link_thumbnailer/scrapers/opengraph/base'
4
+
5
+ module LinkThumbnailer
6
+ module Scrapers
7
+ module Opengraph
8
+ class Favicon < ::LinkThumbnailer::Scrapers::Opengraph::Base
9
+
10
+ def value
11
+ nil
12
+ end
13
+
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,107 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'link_thumbnailer/scrapers/opengraph/base'
4
+ require 'link_thumbnailer/uri'
5
+
6
+ module LinkThumbnailer
7
+ module Scrapers
8
+ module Opengraph
9
+ class Image < ::LinkThumbnailer::Scrapers::Opengraph::Base
10
+
11
+ def value
12
+ ::LinkThumbnailer::Scrapers::Opengraph::Image::Base.new(document, website).value +
13
+ ::LinkThumbnailer::Scrapers::Opengraph::Image::Url.new(document, website).value
14
+ end
15
+
16
+ private
17
+
18
+ # Handles `og:image` attributes.
19
+ class Base < ::LinkThumbnailer::Scrapers::Opengraph::Base
20
+
21
+ def value
22
+ model
23
+ end
24
+
25
+ def model
26
+ nodes.map do |n|
27
+ uri = LinkThumbnailer::URI.new(n.attributes['content'])
28
+ modelize(n, uri.to_s) if uri.valid?
29
+ end.compact
30
+ end
31
+
32
+ def modelize(node, text = nil)
33
+ model_class.new(text, size)
34
+ end
35
+
36
+ def model_class
37
+ ::LinkThumbnailer::Models::Image
38
+ end
39
+
40
+ def nodes
41
+ nodes = meta_xpaths(attribute: attribute)
42
+ nodes.empty? ? meta_xpaths(attribute: attribute, key: :name) : nodes
43
+ end
44
+
45
+ def attribute
46
+ 'og:image'
47
+ end
48
+
49
+ def size
50
+ [width.to_i, height.to_i] if width && height
51
+ end
52
+
53
+ def width
54
+ ::LinkThumbnailer::Scrapers::Opengraph::Image::Width.new(document).value
55
+ end
56
+
57
+ def height
58
+ ::LinkThumbnailer::Scrapers::Opengraph::Image::Height.new(document).value
59
+ end
60
+
61
+ end
62
+
63
+ # Handles `og:image:url` attributes.
64
+ class Url < ::LinkThumbnailer::Scrapers::Opengraph::Image::Base
65
+
66
+ private
67
+
68
+ def attribute
69
+ 'og:image:url'
70
+ end
71
+
72
+ end
73
+
74
+ # Handles `og:image:width` attributes.
75
+ class Width < ::LinkThumbnailer::Scrapers::Opengraph::Base
76
+
77
+ def value
78
+ node.attributes['content'].to_s if node
79
+ end
80
+
81
+ private
82
+
83
+ def attribute
84
+ 'og:image:width'
85
+ end
86
+
87
+ end
88
+
89
+ # Handles `og:image:height` attributes.
90
+ class Height < ::LinkThumbnailer::Scrapers::Opengraph::Base
91
+
92
+ def value
93
+ node.attributes['content'].to_s if node
94
+ end
95
+
96
+ private
97
+
98
+ def attribute
99
+ 'og:image:height'
100
+ end
101
+
102
+ end
103
+
104
+ end
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'link_thumbnailer/scrapers/opengraph/base'
4
+ require 'link_thumbnailer/scrapers/opengraph/image'
5
+
6
+ module LinkThumbnailer
7
+ module Scrapers
8
+ module Opengraph
9
+ class Images < ::LinkThumbnailer::Scrapers::Opengraph::Base
10
+
11
+ def call(attribute_name)
12
+ ::LinkThumbnailer::Scrapers::Opengraph::Image.new(document, website).call('image')
13
+ end
14
+
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'link_thumbnailer/scrapers/opengraph/base'
4
+
5
+ module LinkThumbnailer
6
+ module Scrapers
7
+ module Opengraph
8
+ class Title < ::LinkThumbnailer::Scrapers::Opengraph::Base
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'link_thumbnailer/scrapers/opengraph/base'
4
+
5
+ module LinkThumbnailer
6
+ module Scrapers
7
+ module Opengraph
8
+ class Video < ::LinkThumbnailer::Scrapers::Opengraph::Base
9
+
10
+ def value
11
+ ::LinkThumbnailer::Scrapers::Opengraph::Video::Base.new(document, website).value +
12
+ ::LinkThumbnailer::Scrapers::Opengraph::Video::Url.new(document, website).value
13
+ end
14
+
15
+ private
16
+
17
+ # Handles `og:video` attributes.
18
+ class Base < ::LinkThumbnailer::Scrapers::Opengraph::Base
19
+
20
+ def value
21
+ model
22
+ end
23
+
24
+ private
25
+
26
+ def model
27
+ nodes.map { |n| modelize(n, n.attributes['content'].to_s) }
28
+ end
29
+
30
+ def modelize(node, text = nil)
31
+ model_class.new(text, size)
32
+ end
33
+
34
+ def model_class
35
+ ::LinkThumbnailer::Models::Video
36
+ end
37
+
38
+ def nodes
39
+ nodes = meta_xpaths(attribute: attribute)
40
+ nodes.empty? ? meta_xpaths(attribute: attribute, key: :name) : nodes
41
+ end
42
+
43
+ def attribute
44
+ return 'og:url' if vimeo?
45
+ 'og:video'
46
+ end
47
+
48
+ # Vimeo uses a SWF file for its og:video property which doesn't
49
+ # provide any metadata for the VideoInfo gem downstream. Using
50
+ # og:url means VideoInfo is passed a webpage URL with metadata
51
+ # it can parse.
52
+ def vimeo?
53
+ website.url.host =~ /vimeo/
54
+ end
55
+
56
+ def size
57
+ [width.to_i, height.to_i] if width && height
58
+ end
59
+
60
+ def width
61
+ ::LinkThumbnailer::Scrapers::Opengraph::Video::Width.new(document).value
62
+ end
63
+
64
+ def height
65
+ ::LinkThumbnailer::Scrapers::Opengraph::Video::Height.new(document).value
66
+ end
67
+
68
+ end
69
+
70
+ # Handles `og:video:url` attributes.
71
+ class Url < ::LinkThumbnailer::Scrapers::Opengraph::Video::Base
72
+
73
+ private
74
+
75
+ def attribute
76
+ super
77
+ 'og:video:url'
78
+ end
79
+
80
+ end
81
+
82
+ # Handles `og:video:width` attributes.
83
+ class Width < ::LinkThumbnailer::Scrapers::Opengraph::Base
84
+
85
+ def value
86
+ node.attributes['content'].to_s if node
87
+ end
88
+
89
+ private
90
+
91
+ def attribute
92
+ 'og:video:width'
93
+ end
94
+
95
+ end
96
+
97
+ # Handles `og:video:height` attributes.
98
+ class Height < ::LinkThumbnailer::Scrapers::Opengraph::Base
99
+
100
+ def value
101
+ node.attributes['content'].to_s if node
102
+ end
103
+
104
+ private
105
+
106
+ def attribute
107
+ 'og:video:height'
108
+ end
109
+
110
+ end
111
+
112
+ end
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'link_thumbnailer/scrapers/opengraph/base'
4
+ require 'link_thumbnailer/scrapers/opengraph/video'
5
+
6
+ module LinkThumbnailer
7
+ module Scrapers
8
+ module Opengraph
9
+ class Videos < ::LinkThumbnailer::Scrapers::Opengraph::Base
10
+
11
+ def call(attribute_name)
12
+ ::LinkThumbnailer::Scrapers::Opengraph::Video.new(document, website).call('video')
13
+ end
14
+
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module LinkThumbnailer
4
+ class URI
5
+
6
+ attr_reader :attribute
7
+
8
+ def initialize(uri)
9
+ @attribute = uri.to_s
10
+ end
11
+
12
+ def valid?
13
+ !!(attribute =~ ::URI::regexp)
14
+ end
15
+
16
+ def to_s
17
+ attribute
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module LinkThumbnailer
4
+ VERSION = '3.3.2'
5
+ end
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'video_info'
4
+
5
+ module LinkThumbnailer
6
+ class VideoParser
7
+
8
+ attr_reader :parser
9
+
10
+ def initialize(video)
11
+ @parser = ::VideoInfo.new(video.src.dup.to_s)
12
+ rescue ::VideoInfo::UrlError
13
+ @parser = nil
14
+ end
15
+
16
+ def id
17
+ parser.video_id
18
+ rescue NoMethodError
19
+ nil
20
+ end
21
+
22
+ def size
23
+ [parser.width, parser.height]
24
+ rescue NoMethodError, ::OpenURI::HTTPError
25
+ []
26
+ end
27
+
28
+ def duration
29
+ parser.duration
30
+ rescue NoMethodError, ::OpenURI::HTTPError
31
+ nil
32
+ end
33
+
34
+ def provider
35
+ parser.provider
36
+ rescue NoMethodError, ::OpenURI::HTTPError
37
+ nil
38
+ end
39
+
40
+ def embed_code
41
+ parser.embed_code
42
+ rescue NoMethodError, ::OpenURI::HTTPError
43
+ nil
44
+ end
45
+
46
+ end
47
+ end