link_thumbnailer 3.2.0 → 3.4.0

data/lib/link_thumbnailer/models/title.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'link_thumbnailer/model'
 
 module LinkThumbnailer

data/lib/link_thumbnailer/models/video.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'link_thumbnailer/model'
 require 'link_thumbnailer/video_parser'
 

data/lib/link_thumbnailer/models/website.rb

@@ -1,52 +1,54 @@
-require 'link_thumbnailer/model'
-
-module LinkThumbnailer
-  module Models
-    class Website < ::LinkThumbnailer::Model
-
-      attr_accessor :url, :title, :description, :images, :videos, :favicon
-
-      def initialize
-        @images = []
-        @videos = []
-      end
-
-      def video=(video)
-        self.videos = video
-      end
-
-      def videos=(videos)
-        Array(videos).each do |video|
-          @videos << video
-        end
-      end
-
-      def image=(image)
-        self.images = image
-      end
-
-      def images=(images)
-        Array(images).each do |image|
-          next unless image.valid?
-          @images << image
-        end
-      end
-
-      def images
-        @images.sort!
-      end
-
-      def as_json(*)
-        {
-          url:          url.to_s,
-          favicon:      favicon,
-          title:        title,
-          description:  description,
-          images:       images.map(&:as_json),
-          videos:       videos.map(&:as_json)
-        }
-      end
-
-    end
-  end
-end
+# frozen_string_literal: true
+
+require 'link_thumbnailer/model'
+
+module LinkThumbnailer
+  module Models
+    class Website < ::LinkThumbnailer::Model
+
+      attr_accessor :url, :title, :description, :images, :videos, :favicon
+
+      def initialize
+        @images = []
+        @videos = []
+      end
+
+      def video=(video)
+        self.videos = video
+      end
+
+      def videos=(videos)
+        Array(videos).each do |video|
+          @videos << video
+        end
+      end
+
+      def image=(image)
+        self.images = image
+      end
+
+      def images=(images)
+        Array(images).each do |image|
+          next unless image.valid?
+          @images << image
+        end
+      end
+
+      def images
+        @images.sort!
+      end
+
+      def as_json(*)
+        {
+          url:          url.to_s,
+          favicon:      favicon,
+          title:        title,
+          description:  description,
+          images:       images.map(&:as_json),
+          videos:       videos.map(&:as_json)
+        }
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/page.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+require 'link_thumbnailer/response'
 require 'link_thumbnailer/processor'
 require 'link_thumbnailer/scraper'
 
@@ -14,7 +17,7 @@ module LinkThumbnailer
     end
 
     def generate
-      @source = processor.call(url)
+      @source = processor.start(url)
       scraper.call
     end
 

data/lib/link_thumbnailer/parser.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 require 'nokogiri'
 
 module LinkThumbnailer
   class Parser
 
     def call(source)
-      ::Nokogiri::HTML(source)
+      ::Nokogiri::HTML(source, nil, LinkThumbnailer.page.config.encoding)
     rescue ::Nokogiri::XML::SyntaxError => e
       raise ::LinkThumbnailer::SyntaxError.new(e.message)
     end

data/lib/link_thumbnailer/processor.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'delegate'
 require 'uri'
 require 'net/http/persistent'
@@ -15,6 +17,12 @@ module LinkThumbnailer
       super(config)
     end
 
+    def start(url)
+      result = call(url)
+      shutdown
+      result
+    end
+
     def call(url = '', redirect_count = 0, headers = {})
       self.url        = url
       @redirect_count = redirect_count
@@ -26,12 +34,16 @@ module LinkThumbnailer
         set_http_options
         perform_request
       end
-    rescue ::Net::HTTPExceptions, ::SocketError, ::Timeout::Error => e
+    rescue ::Net::HTTPExceptions, ::SocketError, ::Timeout::Error, ::Net::HTTP::Persistent::Error => e
       raise ::LinkThumbnailer::HTTPError.new(e.message)
     end
 
     private
 
+    def shutdown
+      http.shutdown
+    end
+
     def with_valid_url
       raise ::LinkThumbnailer::BadUriFormat unless valid_url_format?
       yield if block_given?
@@ -39,8 +51,8 @@ module LinkThumbnailer
 
     def set_http_headers(headers = {})
       headers.each { |k, v| http.headers[k] = v }
-      http.headers['User-Agent']               = user_agent
-      http.override_headers['Accept-Encoding'] = 'none'
+      http.override_headers['User-Agent'] = user_agent
+      config.http_override_headers.each { |k, v| http.override_headers[k] = v }
     end
 
     def set_http_options
@@ -51,7 +63,7 @@ module LinkThumbnailer
     end
 
     def perform_request
-      response          = http.request(url)
+      response          = request_in_chunks
       headers           = {}
       headers['Cookie'] = response['Set-Cookie'] if response['Set-Cookie'].present?
 
@@ -59,7 +71,7 @@ module LinkThumbnailer
 
       case response
       when ::Net::HTTPSuccess
-        response.body
+        Response.new(response).body
       when ::Net::HTTPRedirection
         call(
           resolve_relative_url(response['location'].to_s),
@@ -71,6 +83,19 @@ module LinkThumbnailer
       end
     end
 
+    def request_in_chunks
+      body     = String.new
+      response = http.request(url) do |resp|
+        raise ::LinkThumbnailer::DownloadSizeLimit if too_big_download_size?(resp.content_length)
+        resp.read_body do |chunk|
+          body.concat(chunk)
+          raise ::LinkThumbnailer::DownloadSizeLimit if too_big_download_size?(body.length)
+        end
+      end
+      response.body = body
+      response
+    end
+
     def resolve_relative_url(location)
       location.start_with?('http') ? location : build_absolute_url_for(location)
     end
@@ -99,6 +124,10 @@ module LinkThumbnailer
       config.verify_ssl
     end
 
+    def download_size_limit
+      config.download_size_limit
+    end
+
     def too_many_redirections?
       redirect_count > redirect_limit
     end
@@ -118,6 +147,10 @@ module LinkThumbnailer
       false
     end
 
+    def too_big_download_size?(size)
+      size.to_i > download_size_limit.to_i
+    end
+
     def url=(url)
       @url = ::URI.parse(url.to_s)
     end

data/lib/link_thumbnailer/railtie.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LinkThumbnailer
   class Railtie < ::Rails::Railtie
   end

data/lib/link_thumbnailer/response.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module LinkThumbnailer
+  class Response
+    def initialize(response)
+      @response = response
+    end
+
+    def charset
+      @charset ||= extract_charset
+    end
+
+    def body
+      @body ||= extract_body
+    end
+
+    private
+
+    def extract_charset
+      content_type = @response['Content-Type'] || ''
+      m = content_type.match(/charset=([\w-]+)/)
+      (m && m[1]) || @response.body.scrub =~ /<meta[^>]*charset\s*=\s*["']?(.+?)["' >]/i && $1 || ''
+    end
+
+    def extract_body
+      should_convert_body_to_utf8? ? convert_encoding_to_utf8(@response.body, charset) : @response.body
+    end
+
+    def should_convert_body_to_utf8?
+      charset != '' && charset != 'utf-8'
+    end
+
+    def convert_encoding_to_utf8(body, from)
+      Encoding::Converter.new(from, 'utf-8').convert(body)
+    rescue EncodingError
+      body
+    end
+  end
+end

data/lib/link_thumbnailer/scraper.rb

@@ -1,60 +1,62 @@
-require 'delegate'
-require 'active_support/core_ext/object/blank'
-require 'active_support/inflector'
-
-require 'link_thumbnailer/parser'
-require 'link_thumbnailer/models/website'
-require 'link_thumbnailer/scrapers/default/title'
-require 'link_thumbnailer/scrapers/opengraph/title'
-require 'link_thumbnailer/scrapers/default/description'
-require 'link_thumbnailer/scrapers/opengraph/description'
-require 'link_thumbnailer/scrapers/default/images'
-require 'link_thumbnailer/scrapers/opengraph/images'
-require 'link_thumbnailer/scrapers/default/videos'
-require 'link_thumbnailer/scrapers/opengraph/videos'
-require 'link_thumbnailer/scrapers/default/favicon'
-require 'link_thumbnailer/scrapers/opengraph/favicon'
-
-module LinkThumbnailer
-  class Scraper < ::SimpleDelegator
-
-    attr_reader :document, :source, :url, :config, :website
-
-    def initialize(source, url)
-      @source       = source
-      @url          = url
-      @config       = ::LinkThumbnailer.page.config
-      @document     = parser.call(source)
-      @website      = ::LinkThumbnailer::Models::Website.new
-      @website.url  = url
-
-      super(config)
-    end
-
-    def call
-      config.attributes.each do |name|
-        config.scrapers.each do |scraper_prefix|
-          scraper_class(scraper_prefix, name).new(document, website).call(name.to_s)
-          break unless website.send(name).blank?
-        end
-      end
-
-      website
-    end
-
-    private
-
-    def scraper_class(prefix, name)
-      prefix = "::LinkThumbnailer::Scrapers::#{prefix.to_s.camelize}"
-      name = name.to_s.camelize
-      "#{prefix}::#{name}".constantize
-    rescue NameError
-      raise ::LinkThumbnailer::ScraperInvalid, "scraper named '#{prefix}::#{name}' does not exists."
-    end
-
-    def parser
-      ::LinkThumbnailer::Parser.new
-    end
-
-  end
-end
+# frozen_string_literal: true
+
+require 'delegate'
+require 'active_support/core_ext/object/blank'
+require 'active_support/inflector'
+
+require 'link_thumbnailer/parser'
+require 'link_thumbnailer/models/website'
+require 'link_thumbnailer/scrapers/default/title'
+require 'link_thumbnailer/scrapers/opengraph/title'
+require 'link_thumbnailer/scrapers/default/description'
+require 'link_thumbnailer/scrapers/opengraph/description'
+require 'link_thumbnailer/scrapers/default/images'
+require 'link_thumbnailer/scrapers/opengraph/images'
+require 'link_thumbnailer/scrapers/default/videos'
+require 'link_thumbnailer/scrapers/opengraph/videos'
+require 'link_thumbnailer/scrapers/default/favicon'
+require 'link_thumbnailer/scrapers/opengraph/favicon'
+
+module LinkThumbnailer
+  class Scraper < ::SimpleDelegator
+
+    attr_reader :document, :source, :url, :config, :website
+
+    def initialize(source, url)
+      @source       = source
+      @url          = url
+      @config       = ::LinkThumbnailer.page.config
+      @document     = parser.call(source)
+      @website      = ::LinkThumbnailer::Models::Website.new
+      @website.url  = url
+
+      super(config)
+    end
+
+    def call
+      config.attributes.each do |name|
+        config.scrapers.each do |scraper_prefix|
+          scraper_class(scraper_prefix, name).new(document, website).call(name.to_s)
+          break unless website.send(name).blank?
+        end
+      end
+
+      website
+    end
+
+    private
+
+    def scraper_class(prefix, name)
+      prefix = "::LinkThumbnailer::Scrapers::#{prefix.to_s.camelize}"
+      name = name.to_s.camelize
+      "#{prefix}::#{name}".constantize
+    rescue NameError
+      raise ::LinkThumbnailer::ScraperInvalid, "scraper named '#{prefix}::#{name}' does not exists."
+    end
+
+    def parser
+      ::LinkThumbnailer::Parser.new
+    end
+
+  end
+end

data/lib/link_thumbnailer/scrapers/base.rb

@@ -1,67 +1,69 @@
-require 'delegate'
-require 'link_thumbnailer/models/title'
-require 'link_thumbnailer/models/description'
-require 'link_thumbnailer/models/image'
-require 'link_thumbnailer/models/video'
-
-module LinkThumbnailer
-  module Scrapers
-    class Base < ::SimpleDelegator
-
-      attr_reader :config, :document, :website, :attribute_name
-
-      def initialize(document, website = nil)
-        @config   = ::LinkThumbnailer.page.config
-        @document = document
-        @website  = website
-
-        super(config)
-      end
-
-      def call(attribute_name)
-        return false unless website.present?
-        return false unless applicable?
-
-        @attribute_name = attribute_name
-
-        website.send("#{attribute_name}=", value)
-        website
-      end
-
-      def applicable?
-        true
-      end
-
-      def value
-        fail NotImplementedError
-      end
-
-      private
-
-      def meta_xpath(options = {})
-        meta_xpaths(options).first
-      end
-
-      def meta_xpaths(options = {})
-        key       = options.fetch(:key, :property)
-        value     = options.fetch(:value, :content)
-        attribute = options.fetch(:attribute, attribute_name)
-
-        document.xpath("//meta[translate(@#{key},'#{abc.upcase}','#{abc}') = '#{attribute}' and @#{value}]")
-      end
-
-      def abc
-        'abcdefghijklmnopqrstuvwxyz'
-      end
-
-      def model_class
-        "::LinkThumbnailer::Models::#{attribute_name.to_s.camelize}".constantize
-      end
-
-      def modelize(node, text = nil)
-        model_class.new(node, text)
-      end
-
-    end
-  end
-end
+# frozen_string_literal: true
+
+require 'delegate'
+require 'link_thumbnailer/models/title'
+require 'link_thumbnailer/models/description'
+require 'link_thumbnailer/models/image'
+require 'link_thumbnailer/models/video'
+
+module LinkThumbnailer
+  module Scrapers
+    class Base < ::SimpleDelegator
+
+      attr_reader :config, :document, :website, :attribute_name
+
+      def initialize(document, website = nil)
+        @config   = ::LinkThumbnailer.page.config
+        @document = document
+        @website  = website
+
+        super(config)
+      end
+
+      def call(attribute_name)
+        return false unless website.present?
+        return false unless applicable?
+
+        @attribute_name = attribute_name
+
+        website.send("#{attribute_name}=", value)
+        website
+      end
+
+      def applicable?
+        true
+      end
+
+      def value
+        fail NotImplementedError
+      end
+
+      private
+
+      def meta_xpath(options = {})
+        meta_xpaths(options).first
+      end
+
+      def meta_xpaths(options = {})
+        key       = options.fetch(:key, :property)
+        value     = options.fetch(:value, :content)
+        attribute = options.fetch(:attribute, attribute_name)
+
+        document.xpath("//meta[translate(@#{key},'#{abc.upcase}','#{abc}') = '#{attribute}' and string-length(@#{value}) > 0]")
+      end
+
+      def abc
+        'abcdefghijklmnopqrstuvwxyz'
+      end
+
+      def model_class
+        "::LinkThumbnailer::Models::#{attribute_name.to_s.camelize}".constantize
+      end
+
+      def modelize(node, text = nil)
+        model_class.new(node, text)
+      end
+
+    end
+  end
+end