link_thumbnailer 3.2.0 → 3.4.0

data/Gemfile

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in link_thumbnailer.gemspec
 gemspec
 
 group :development, :test do
-  gem 'rspec',      '~> 2.14'
-  gem 'webmock',    '~> 1.14'
-  gem 'pry',        '~> 0.9'
+  gem 'rspec',      '>= 2.14'
+  gem 'webmock',    '>= 1.14'
+  gem 'pry',        '>= 0.9'
 end

data/README.md

@@ -165,6 +165,10 @@ LinkThumbnailer.configure do |config|
   # Sets number of concurrent http connections that can be opened to fetch images informations such as size and type.
   #
   # config.max_concurrency = 20
+
+  # Sets the default encoding.
+  #
+  # config.encoding = 'utf-8'
 end
 ```
 

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 

data/lib/generators/link_thumbnailer/install_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LinkThumbnailer
   module Generators
     class InstallGenerator < ::Rails::Generators::Base

data/lib/generators/templates/initializer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Use this hook to configure LinkThumbnailer bahaviors.
 LinkThumbnailer.configure do |config|
   # Numbers of redirects before raising an exception when trying to parse given url.
@@ -33,6 +35,11 @@ LinkThumbnailer.configure do |config|
   #
   # config.attributes = [:title, :images, :description, :videos, :favicon]
 
+  # Prior favicon size. If the website doesn't have such size - returns the first favicon.
+  # Value should be like '32x32' or '16x16'. Default value is nil.
+  #
+  # config.favicon_size = nil
+
   # List of procedures used to rate the website description. Add you custom class
   # here. See wiki for more details on how to build your own graders.
   #
@@ -80,4 +87,12 @@ LinkThumbnailer.configure do |config|
   # Defines the strategies to use to scrap the website. See the [Open Graph Protocol](http://ogp.me/) for more information.
   #
   # config.scrapers = [:opengraph, :default]
+
+  # Limit for download size in bytes. When using ActiveSupport, you can also use values like 10.megabytes
+  #
+  # config.download_size_limit = 10 * 1024 * 1024
+
+  # Sets the default encoding.
+  #
+  # config.encoding = 'utf-8'
 end

data/lib/link_thumbnailer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'link_thumbnailer/version'
 require 'link_thumbnailer/configuration'

data/lib/link_thumbnailer/configuration.rb

@@ -1,68 +1,74 @@
-module LinkThumbnailer
-
-  # Access point for the gem configurations.
-  #
-  # @return [LinkThumbnailer::Configuration] a configuration instance.
-  def self.config
-    @config ||= Configuration.new
-  end
-
-  # Configure hook used in the gem initializer. Convinient way to set all the
-  # gem configurations.
-  #
-  # @example inside config/initializers/link_thumbnaler.rb
-  #   LinkThumbnailer.configure do |config|
-  #     config.user_agent = 'link_thumbnailer'
-  #   end
-  #
-  # @return [void]
-  def self.configure
-    yield config if block_given?
-  end
-
-	class Configuration
-
-    attr_accessor :redirect_limit, :blacklist_urls, :user_agent,
-                  :verify_ssl, :http_open_timeout, :http_read_timeout, :attributes,
-                  :graders, :description_min_length, :positive_regex, :negative_regex,
-                  :image_limit, :image_stats, :raise_on_invalid_format, :max_concurrency,
-                  :scrapers
-
-    alias_method :http_timeout, :http_open_timeout
-    alias_method :http_timeout=, :http_open_timeout=
-
-    # Create a new instance.
-    #
-    # @return [LinkThumbnailer::Configuration]
-    def initialize
-      @redirect_limit         = 3
-      @user_agent             = 'link_thumbnailer'
-      @verify_ssl             = true
-      @http_open_timeout      = 5
-      @http_read_timeout      = 5
-      @blacklist_urls         = [
-        %r{^http://ad\.doubleclick\.net/},
-        %r{^http://b\.scorecardresearch\.com/},
-        %r{^http://pixel\.quantserve\.com/},
-        %r{^http://s7\.addthis\.com/}
-      ]
-      @attributes             = [:title, :images, :description, :videos, :favicon]
-      @graders                = [
-        ->(description) { ::LinkThumbnailer::Graders::Length.new(description) },
-        ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :class) },
-        ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :id) },
-        ->(description) { ::LinkThumbnailer::Graders::Position.new(description, weigth: 3) },
-        ->(description) { ::LinkThumbnailer::Graders::LinkDensity.new(description) },
-      ]
-      @description_min_length = 50
-      @positive_regex = /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i
-      @negative_regex = /combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget|modal/i
-      @image_limit    = 5
-      @image_stats    = true
-      @raise_on_invalid_format = false
-      @max_concurrency = 20
-      @scrapers = [:opengraph, :default]
-    end
-
-	end
-end
+# frozen_string_literal: true
+
+module LinkThumbnailer
+
+  # Access point for the gem configurations.
+  #
+  # @return [LinkThumbnailer::Configuration] a configuration instance.
+  def self.config
+    @config ||= Configuration.new
+  end
+
+  # Configure hook used in the gem initializer. Convinient way to set all the
+  # gem configurations.
+  #
+  # @example inside config/initializers/link_thumbnaler.rb
+  #   LinkThumbnailer.configure do |config|
+  #     config.user_agent = 'link_thumbnailer'
+  #   end
+  #
+  # @return [void]
+  def self.configure
+    yield config if block_given?
+  end
+
+	class Configuration
+
+    attr_accessor :redirect_limit, :blacklist_urls, :user_agent,
+                  :verify_ssl, :http_open_timeout, :http_read_timeout, :attributes,
+                  :graders, :description_min_length, :positive_regex, :negative_regex,
+                  :image_limit, :image_stats, :raise_on_invalid_format, :max_concurrency,
+                  :scrapers, :http_override_headers, :download_size_limit, :encoding,
+                  :favicon_size
+
+    alias_method :http_timeout, :http_open_timeout
+    alias_method :http_timeout=, :http_open_timeout=
+
+    # Create a new instance.
+    #
+    # @return [LinkThumbnailer::Configuration]
+    def initialize
+      @redirect_limit         = 3
+      @user_agent             = 'link_thumbnailer'
+      @verify_ssl             = true
+      @http_open_timeout      = 5
+      @http_read_timeout      = 5
+      @blacklist_urls         = [
+        %r{^http://ad\.doubleclick\.net/},
+        %r{^http://b\.scorecardresearch\.com/},
+        %r{^http://pixel\.quantserve\.com/},
+        %r{^http://s7\.addthis\.com/}
+      ]
+      @attributes             = [:title, :images, :description, :videos, :favicon]
+      @graders                = [
+        ->(description) { ::LinkThumbnailer::Graders::Length.new(description) },
+        ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :class) },
+        ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :id) },
+        ->(description) { ::LinkThumbnailer::Graders::Position.new(description, weigth: 3) },
+        ->(description) { ::LinkThumbnailer::Graders::LinkDensity.new(description) },
+      ]
+      @description_min_length = 50
+      @positive_regex = /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i
+      @negative_regex = /combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget|modal/i
+      @image_limit    = 5
+      @image_stats    = true
+      @raise_on_invalid_format = false
+      @max_concurrency = 20
+      @scrapers = [:opengraph, :default]
+      @http_override_headers = { 'Accept-Encoding' => 'none' }
+      @download_size_limit = 10 * 1024 * 1024
+      @encoding = 'utf-8'
+    end
+
+	end
+end

data/lib/link_thumbnailer/exceptions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LinkThumbnailer
   Exceptions         = Class.new(StandardError)
   RedirectLimit      = Class.new(Exceptions)
@@ -6,4 +8,5 @@ module LinkThumbnailer
   ScraperInvalid     = Class.new(Exceptions)
   HTTPError          = Class.new(Exceptions)
   SyntaxError        = Class.new(Exceptions)
+  DownloadSizeLimit  = Class.new(Exceptions)
 end

data/lib/link_thumbnailer/grader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'delegate'
 require 'link_thumbnailer/graders/base'
 require 'link_thumbnailer/graders/length'

data/lib/link_thumbnailer/graders/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'delegate'
 
 module LinkThumbnailer

data/lib/link_thumbnailer/graders/html_attribute.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LinkThumbnailer
   module Graders
     class HtmlAttribute < ::LinkThumbnailer::Graders::Base

data/lib/link_thumbnailer/graders/length.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LinkThumbnailer
   module Graders
     class Length < ::LinkThumbnailer::Graders::Base

data/lib/link_thumbnailer/graders/link_density.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LinkThumbnailer
   module Graders
     class LinkDensity < ::LinkThumbnailer::Graders::Base

data/lib/link_thumbnailer/graders/position.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LinkThumbnailer
   module Graders
     class Position < ::LinkThumbnailer::Graders::Base

data/lib/link_thumbnailer/image_comparator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'link_thumbnailer/image_comparators/base'
 require 'link_thumbnailer/image_comparators/size'
 

data/lib/link_thumbnailer/image_comparators/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LinkThumbnailer
   module ImageComparators
     class Base

data/lib/link_thumbnailer/image_comparators/size.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LinkThumbnailer
   module ImageComparators
     class Size < ::LinkThumbnailer::ImageComparators::Base

data/lib/link_thumbnailer/image_parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'image_info'
 
 module LinkThumbnailer
@@ -6,7 +8,7 @@ module LinkThumbnailer
     attr_reader :images
 
     def initialize(urls)
-      @images = perform? ? ::ImageInfo.from(urls, max_concurrency: max_concurrency) : Array(urls).map(&method(:build_default_image))
+      @images = perform? ? image_info(urls) : default_images(urls)
     end
 
     def size
@@ -19,6 +21,10 @@ module LinkThumbnailer
 
     private
 
+    def default_images(urls)
+      Array(urls).compact.map(&method(:build_default_image))
+    end
+
     def build_default_image(uri)
       NullImage.new(uri)
     end
@@ -31,6 +37,12 @@ module LinkThumbnailer
       ::LinkThumbnailer.page.config.max_concurrency
     end
 
+    def image_info(urls)
+      ::ImageInfo.from(urls, max_concurrency: max_concurrency)
+    rescue
+      default_images(urls)
+    end
+
     class NullImage
       attr_reader :uri
 

data/lib/link_thumbnailer/image_validator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'delegate'
 
 module LinkThumbnailer

data/lib/link_thumbnailer/model.rb

@@ -1,17 +1,20 @@
-module LinkThumbnailer
-  class Model
-
-    def to_json(*args)
-      as_json.to_json(*args)
-    end
-
-    private
-
-    def sanitize(str)
-      return unless str
-
-      str.encode!("UTF-16", "UTF-8", invalid: :replace, undef: :replace, replace: "")
-      str.encode!("UTF-8", "UTF-16").strip.gsub(/[\r\n\f]+/, "\n")
-    end
-  end
-end
+# frozen_string_literal: true
+
+module LinkThumbnailer
+  class Model
+
+    def to_json(*args)
+      as_json.to_json(*args)
+    end
+
+    private
+
+    def sanitize(str)
+      return unless str
+
+      str = str.encode("UTF-16", "UTF-8", invalid: :replace, undef: :replace, replace: "")
+      str = str.encode("UTF-8", "UTF-16").strip.gsub(/[\r\n\f]+/, "\n")
+      str
+    end
+  end
+end

data/lib/link_thumbnailer/models/description.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'link_thumbnailer/model'
 require 'link_thumbnailer/grader'
 

data/lib/link_thumbnailer/models/favicon.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'link_thumbnailer/model'
 
 module LinkThumbnailer

data/lib/link_thumbnailer/models/image.rb

@@ -1,54 +1,56 @@
-require 'link_thumbnailer/model'
-require 'link_thumbnailer/image_parser'
-require 'link_thumbnailer/image_comparator'
-require 'link_thumbnailer/image_validator'
-
-module LinkThumbnailer
-  module Models
-    class Image < ::LinkThumbnailer::Model
-
-      attr_reader :src, :type, :size
-
-      def initialize(src, size = nil, type = nil)
-        @src  = src
-        @size = size || parser.size
-        @type = type || parser.type
-      end
-
-      def to_s
-        src.to_s
-      end
-
-      def <=>(other)
-        comparator.call(other)
-      end
-
-      def valid?
-        validator.call
-      end
-
-      def as_json(*)
-        {
-          src:  src.to_s,
-          size: size,
-          type: type
-        }
-      end
-
-      private
-
-      def parser
-        @parser ||= ::LinkThumbnailer::ImageParser.new(src)
-      end
-
-      def validator
-        ::LinkThumbnailer::ImageValidator.new(self)
-      end
-
-      def comparator
-        ::LinkThumbnailer::ImageComparator.new(self)
-      end
-
-    end
-  end
-end
+# frozen_string_literal: true
+
+require 'link_thumbnailer/model'
+require 'link_thumbnailer/image_parser'
+require 'link_thumbnailer/image_comparator'
+require 'link_thumbnailer/image_validator'
+
+module LinkThumbnailer
+  module Models
+    class Image < ::LinkThumbnailer::Model
+
+      attr_reader :src, :type, :size
+
+      def initialize(src, size = nil, type = nil)
+        @src  = src
+        @size = size || parser.size
+        @type = type || parser.type
+      end
+
+      def to_s
+        src.to_s
+      end
+
+      def <=>(other)
+        comparator.call(other)
+      end
+
+      def valid?
+        validator.call
+      end
+
+      def as_json(*)
+        {
+          src:  src.to_s,
+          size: size,
+          type: type
+        }
+      end
+
+      private
+
+      def parser
+        @parser ||= ::LinkThumbnailer::ImageParser.new(src)
+      end
+
+      def validator
+        ::LinkThumbnailer::ImageValidator.new(self)
+      end
+
+      def comparator
+        ::LinkThumbnailer::ImageComparator.new(self)
+      end
+
+    end
+  end
+end