link_thumbnailer 3.3.2

data/lib/link_thumbnailer/parser.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module LinkThumbnailer
+  class Parser
+
+    def call(source)
+      ::Nokogiri::HTML(source, nil, LinkThumbnailer.page.config.encoding)
+    rescue ::Nokogiri::XML::SyntaxError => e
+      raise ::LinkThumbnailer::SyntaxError.new(e.message)
+    end
+
+  end
+end

data/lib/link_thumbnailer/processor.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require 'delegate'
+require 'uri'
+require 'net/http/persistent'
+
+module LinkThumbnailer
+  class Processor < ::SimpleDelegator
+
+    attr_accessor :url
+    attr_reader   :config, :http, :redirect_count
+
+    def initialize
+      @config = ::LinkThumbnailer.page.config
+      @http   = ::Net::HTTP::Persistent.new
+
+      super(config)
+    end
+
+    def call(url = '', redirect_count = 0, headers = {})
+      self.url        = url
+      @redirect_count = redirect_count
+
+      raise ::LinkThumbnailer::RedirectLimit if too_many_redirections?
+
+      with_valid_url do
+        set_http_headers(headers)
+        set_http_options
+        perform_request
+      end
+    rescue ::Net::HTTPExceptions, ::SocketError, ::Timeout::Error => e
+      raise ::LinkThumbnailer::HTTPError.new(e.message)
+    end
+
+    private
+
+    def with_valid_url
+      raise ::LinkThumbnailer::BadUriFormat unless valid_url_format?
+      yield if block_given?
+    end
+
+    def set_http_headers(headers = {})
+      headers.each { |k, v| http.headers[k] = v }
+      http.override_headers['User-Agent'] = user_agent
+      config.http_override_headers.each { |k, v| http.override_headers[k] = v }
+    end
+
+    def set_http_options
+      http.verify_mode  = ::OpenSSL::SSL::VERIFY_NONE unless ssl_required?
+      http.open_timeout = http_open_timeout
+      http.read_timeout = http_read_timeout
+      http.proxy = :ENV
+    end
+
+    def perform_request
+      response          = http.request(url)
+      headers           = {}
+      headers['Cookie'] = response['Set-Cookie'] if response['Set-Cookie'].present?
+
+      raise ::LinkThumbnailer::FormatNotSupported.new(response['Content-Type']) unless valid_response_format?(response)
+
+      case response
+      when ::Net::HTTPSuccess
+        Response.new(response).body
+      when ::Net::HTTPRedirection
+        call(
+          resolve_relative_url(response['location'].to_s),
+          redirect_count + 1,
+          headers
+        )
+      else
+        response.error!
+      end
+    end
+
+    def resolve_relative_url(location)
+      location.start_with?('http') ? location : build_absolute_url_for(location)
+    end
+
+    def build_absolute_url_for(relative_url)
+      ::URI.parse("#{url.scheme}://#{url.host}#{relative_url}")
+    end
+
+    def redirect_limit
+      config.redirect_limit
+    end
+
+    def user_agent
+      config.user_agent
+    end
+
+    def http_open_timeout
+      config.http_open_timeout
+    end
+
+    def http_read_timeout
+      config.http_read_timeout
+    end
+
+    def ssl_required?
+      config.verify_ssl
+    end
+
+    def too_many_redirections?
+      redirect_count > redirect_limit
+    end
+
+    def valid_url_format?
+      url.is_a?(::URI::HTTP)
+    end
+
+    def valid_response_format?(response)
+      return true unless config.raise_on_invalid_format
+      return true if response['Content-Type'] =~ /text\/html/
+      return true if response['Content-Type'] =~ /application\/html/
+      return true if response['Content-Type'] =~ /application\/xhtml\+xml/
+      return true if response['Content-Type'] =~ /application\/xml/
+      return true if response['Content-Type'] =~ /text\/xml/
+      return true if response['Content-Type'] =~ /text\/plain/
+      false
+    end
+
+    def url=(url)
+      @url = ::URI.parse(url.to_s)
+    end
+
+  end
+end

data/lib/link_thumbnailer/railtie.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module LinkThumbnailer
+  class Railtie < ::Rails::Railtie
+  end
+end

data/lib/link_thumbnailer/response.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module LinkThumbnailer
+  class Response
+    def initialize(response)
+      @response = response
+    end
+
+    def charset
+      @charset ||= extract_charset
+    end
+
+    def body
+      @body ||= extract_body
+    end
+
+    private
+
+    def extract_charset
+      content_type = @response['Content-Type'] || ''
+      m = content_type.match(/charset=(\w+)/)
+      (m && m[1]) || ''
+    end
+
+    def extract_body
+      should_convert_body_to_utf8? ? convert_encoding_to_utf8(@response.body, charset) : @response.body
+    end
+
+    def should_convert_body_to_utf8?
+      charset != '' && charset != 'utf-8'
+    end
+
+    def convert_encoding_to_utf8(body, from)
+      Encoding::Converter.new(from, 'utf-8').convert(body)
+    rescue EncodingError
+      body
+    end
+  end
+end

data/lib/link_thumbnailer/scraper.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'delegate'
+require 'active_support/core_ext/object/blank'
+require 'active_support/inflector'
+
+require 'link_thumbnailer/parser'
+require 'link_thumbnailer/models/website'
+require 'link_thumbnailer/scrapers/default/title'
+require 'link_thumbnailer/scrapers/opengraph/title'
+require 'link_thumbnailer/scrapers/default/description'
+require 'link_thumbnailer/scrapers/opengraph/description'
+require 'link_thumbnailer/scrapers/default/images'
+require 'link_thumbnailer/scrapers/opengraph/images'
+require 'link_thumbnailer/scrapers/default/videos'
+require 'link_thumbnailer/scrapers/opengraph/videos'
+require 'link_thumbnailer/scrapers/default/favicon'
+require 'link_thumbnailer/scrapers/opengraph/favicon'
+
+module LinkThumbnailer
+  class Scraper < ::SimpleDelegator
+
+    attr_reader :document, :source, :url, :config, :website
+
+    def initialize(source, url)
+      @source       = source
+      @url          = url
+      @config       = ::LinkThumbnailer.page.config
+      @document     = parser.call(source)
+      @website      = ::LinkThumbnailer::Models::Website.new
+      @website.url  = url
+
+      super(config)
+    end
+
+    def call
+      config.attributes.each do |name|
+        config.scrapers.each do |scraper_prefix|
+          scraper_class(scraper_prefix, name).new(document, website).call(name.to_s)
+          break unless website.send(name).blank?
+        end
+      end
+
+      website
+    end
+
+    private
+
+    def scraper_class(prefix, name)
+      prefix = "::LinkThumbnailer::Scrapers::#{prefix.to_s.camelize}"
+      name = name.to_s.camelize
+      "#{prefix}::#{name}".constantize
+    rescue NameError
+      raise ::LinkThumbnailer::ScraperInvalid, "scraper named '#{prefix}::#{name}' does not exists."
+    end
+
+    def parser
+      ::LinkThumbnailer::Parser.new
+    end
+
+  end
+end

data/lib/link_thumbnailer/scrapers/base.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'delegate'
+require 'link_thumbnailer/models/title'
+require 'link_thumbnailer/models/description'
+require 'link_thumbnailer/models/image'
+require 'link_thumbnailer/models/video'
+
+module LinkThumbnailer
+  module Scrapers
+    class Base < ::SimpleDelegator
+
+      attr_reader :config, :document, :website, :attribute_name
+
+      def initialize(document, website = nil)
+        @config   = ::LinkThumbnailer.page.config
+        @document = document
+        @website  = website
+
+        super(config)
+      end
+
+      def call(attribute_name)
+        return false unless website.present?
+        return false unless applicable?
+
+        @attribute_name = attribute_name
+
+        website.send("#{attribute_name}=", value)
+        website
+      end
+
+      def applicable?
+        true
+      end
+
+      def value
+        fail NotImplementedError
+      end
+
+      private
+
+      def meta_xpath(options = {})
+        meta_xpaths(options).first
+      end
+
+      def meta_xpaths(options = {})
+        key       = options.fetch(:key, :property)
+        value     = options.fetch(:value, :content)
+        attribute = options.fetch(:attribute, attribute_name)
+
+        document.xpath("//meta[translate(@#{key},'#{abc.upcase}','#{abc}') = '#{attribute}' and string-length(@#{value}) > 0]")
+      end
+
+      def abc
+        'abcdefghijklmnopqrstuvwxyz'
+      end
+
+      def model_class
+        "::LinkThumbnailer::Models::#{attribute_name.to_s.camelize}".constantize
+      end
+
+      def modelize(node, text = nil)
+        model_class.new(node, text)
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/default/base.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'link_thumbnailer/scrapers/base'
+
+module LinkThumbnailer
+  module Scrapers
+    module Default
+      class Base < ::LinkThumbnailer::Scrapers::Base
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/default/description.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'link_thumbnailer/scrapers/default/base'
+
+module LinkThumbnailer
+  module Scrapers
+    module Default
+      class Description < ::LinkThumbnailer::Scrapers::Default::Base
+
+        def value
+          return model_from_meta.to_s if model_from_meta
+          return model_from_body.to_s if model_from_body
+          nil
+        end
+
+        private
+
+        def model_from_meta
+          modelize(node_from_meta, node_from_meta.attributes['content'].value) if node_from_meta
+        end
+
+        def model_from_body
+          nodes_from_body.each_with_index.map { |node, i| modelize(node, node.text, i) }.sort.last
+        end
+
+        def node_from_meta
+          @node_from_meta ||= meta_xpath(key: :name)
+        end
+
+        def nodes_from_body
+          candidates.select { |node| valid_paragraph?(node) }
+        end
+
+        def valid_paragraph?(node)
+          true
+        end
+
+        def candidates
+          document.css('p,td')
+        end
+
+        def modelize(node, text, i = 0)
+          model_class.new(node, text, i, nodes_from_body.count)
+        end
+
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/default/favicon.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'link_thumbnailer/scrapers/default/base'
+require 'link_thumbnailer/models/favicon'
+
+module LinkThumbnailer
+  module Scrapers
+    module Default
+      class Favicon < ::LinkThumbnailer::Scrapers::Default::Base
+
+        def value
+          modelize(to_uri(href)).to_s
+        end
+
+        private
+
+        def to_uri(href)
+          ::URI.parse(href)
+        rescue ::URI::InvalidURIError
+          nil
+        end
+
+        def href
+          node.attributes['href'].value.to_s if node
+        end
+
+        def node
+          document.xpath("//link[contains(@rel, 'icon')]").first
+        end
+
+        def modelize(uri)
+          model_class.new(uri)
+        end
+
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/default/images.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'link_thumbnailer/scrapers/default/base'
+require 'link_thumbnailer/models/image'
+
+module LinkThumbnailer
+  module Scrapers
+    module Default
+      class Images < ::LinkThumbnailer::Scrapers::Default::Base
+
+        def value
+          images.map do |image|
+            modelize(image.uri, image.size, image.type)
+          end
+        end
+
+        private
+
+        def images
+          ::LinkThumbnailer::ImageParser.new(allowed_urls).images
+        end
+
+        def allowed_urls
+          abs_urls.shift(config.image_limit)
+        end
+
+        def urls
+          document.search('//img').map { |i| i['src'] }.compact
+        end
+
+        def abs_urls
+          urls.map do |url|
+            uri = validate_url(url)
+
+            next unless uri
+
+            uri = prefix_uri(uri) if needs_prefix?(uri)
+            uri
+          end
+        end
+
+        def validate_url(url)
+          ::URI.parse(url.to_s)
+        rescue ::URI::InvalidURIError
+          nil
+        end
+
+        def needs_prefix?(uri)
+          !uri.host
+        end
+
+        def prefix_uri(uri)
+          ::URI.join(prefix_url, uri)
+        end
+
+        def prefix_url
+          base_href || website.url
+        end
+
+        def base_href
+          base = document.at('//head/base')
+          base['href'] if base && ::URI.parse(base['href']).host
+        rescue ::URI::InvalidURIError
+          nil
+        end
+
+        def model_class
+          ::LinkThumbnailer::Models::Image
+        end
+
+        def modelize(uri, size = nil, type = nil)
+          model_class.new(uri, size, type)
+        end
+
+      end
+    end
+  end
+end