link_thumbnailer 1.1.2 → 2.0.0

data/lib/link_thumbnailer/page.rb

@@ -0,0 +1,40 @@
+require 'link_thumbnailer/processor'
+require 'link_thumbnailer/scraper'
+
+module LinkThumbnailer
+  class Page
+
+    attr_reader :url, :options, :source
+
+    def initialize(url, options = {})
+      @url     = url
+      @options = options
+
+      set_options
+    end
+
+    def generate
+      @source = processor.call(url)
+      scraper.call
+    end
+
+    def config
+      @config ||= ::LinkThumbnailer.config.dup
+    end
+
+    private
+
+    def set_options
+      options.each { |k, v| config.send("#{k}=", v) }
+    end
+
+    def processor
+      @processor ||= ::LinkThumbnailer::Processor.new
+    end
+
+    def scraper
+      @scraper ||= ::LinkThumbnailer::Scraper.new(source, processor.url)
+    end
+
+  end
+end

data/lib/link_thumbnailer/parser.rb

@@ -0,0 +1,13 @@
+require 'nokogiri'
+
+module LinkThumbnailer
+  class Parser
+
+    attr_reader :document
+
+    def call(source)
+      @document ||= ::Nokogiri::HTML(source)
+    end
+
+  end
+end

data/lib/link_thumbnailer/processor.rb

@@ -0,0 +1,94 @@
+require 'delegate'
+require 'net/http/persistent'
+
+module LinkThumbnailer
+  class Processor < ::SimpleDelegator
+
+    attr_accessor :url
+    attr_reader   :config, :http, :redirect_count
+
+    def initialize
+      @config = ::LinkThumbnailer.page.config
+      @http   = ::Net::HTTP::Persistent.new
+
+      super(config)
+    end
+
+    def call(url = '', redirect_count = 0)
+      self.url        = url
+      @redirect_count = redirect_count
+
+      raise ::LinkThumbnailer::RedirectLimit if too_many_redirections?
+
+      with_valid_url do
+        set_http_headers
+        set_http_options
+        perform_request
+      end
+    end
+
+    private
+
+    def with_valid_url
+      raise ::LinkThumbnailer::BadUriFormat unless valid_url_format?
+      yield if block_given?
+    end
+
+    def set_http_headers
+      http.headers['User-Agent'] = user_agent
+    end
+
+    def set_http_options
+      http.verify_mode  = ::OpenSSL::SSL::VERIFY_NONE unless ssl_required?
+      http.open_timeout = http_timeout
+    end
+
+    def perform_request
+      response = http.request(url)
+      case response
+      when ::Net::HTTPSuccess then response.body
+      when ::Net::HTTPRedirection
+        call resolve_relative_url(response['location']), redirect_count + 1
+      else
+        response.error!
+      end
+    end
+
+    def resolve_relative_url(location)
+      location.start_with?('http') ? location : build_absolute_url_for(location)
+    end
+
+    def build_absolute_url_for(relative_url)
+      URI("#{url.scheme}://#{url.host}#{relative_url}")
+    end
+
+    def redirect_limit
+      config.redirect_limit
+    end
+
+    def user_agent
+      config.user_agent
+    end
+
+    def http_timeout
+      config.http_timeout
+    end
+
+    def ssl_required?
+      config.verify_ssl
+    end
+
+    def too_many_redirections?
+      redirect_count > redirect_limit
+    end
+
+    def valid_url_format?
+      url.is_a?(URI::HTTP)
+    end
+
+    def url=(url)
+      @url = URI(url)
+    end
+
+  end
+end

data/lib/link_thumbnailer/railtie.rb

@@ -1,9 +1,9 @@
-module LinkThumbnailer
-  class Railtie < ::Rails::Railtie
-
-    initializer 'link_thumbnailer.routes' do
-      LinkThumbnailer::Rails::Routes.install!
-    end
-
-  end
-end
+module LinkThumbnailer
+  class Railtie < ::Rails::Railtie
+
+    initializer 'link_thumbnailer.routes' do
+      LinkThumbnailer::Rails::Routes.install!
+    end
+
+  end
+end

data/lib/link_thumbnailer/scraper.rb

@@ -0,0 +1,64 @@
+require 'delegate'
+require 'active_support/core_ext/object/blank'
+require 'active_support/inflector'
+
+require 'link_thumbnailer/parser'
+require 'link_thumbnailer/models/website'
+require 'link_thumbnailer/scrapers/default/title'
+require 'link_thumbnailer/scrapers/opengraph/title'
+require 'link_thumbnailer/scrapers/default/description'
+require 'link_thumbnailer/scrapers/opengraph/description'
+require 'link_thumbnailer/scrapers/default/images'
+require 'link_thumbnailer/scrapers/opengraph/images'
+
+module LinkThumbnailer
+  class Scraper < ::SimpleDelegator
+
+    attr_reader :document, :source, :url, :config, :website
+
+    def initialize(source, url)
+      @source       = source
+      @url          = url
+      @config       = ::LinkThumbnailer.page.config
+      @document     = parser.call(source)
+      @website      = ::LinkThumbnailer::Models::Website.new
+      @website.url  = url
+
+      super(config)
+    end
+
+    def call
+      config.attributes.each do |name|
+        scrapers.each do |scraper_prefix|
+          scraper = scraper_class(scraper_prefix, name).new(document)
+          scraper.call(website, name.to_s) if scraper.applicable?
+
+          break unless website.send(name).blank?
+        end
+      end
+
+      website
+    end
+
+    private
+
+    def scrapers
+      [
+        "::LinkThumbnailer::Scrapers::Opengraph",
+        "::LinkThumbnailer::Scrapers::Default"
+      ]
+    end
+
+    def scraper_class(prefix, name)
+      name = name.to_s.camelize
+      "#{prefix}::#{name}".constantize
+    rescue NameError
+      raise ::LinkThumbnailer::ScraperInvalid, "scraper named '#{prefix}::#{name}' does not exists."
+    end
+
+    def parser
+      ::LinkThumbnailer::Parser.new
+    end
+
+  end
+end

data/lib/link_thumbnailer/scrapers/base.rb

@@ -0,0 +1,63 @@
+require 'delegate'
+require 'link_thumbnailer/models/title'
+require 'link_thumbnailer/models/description'
+require 'link_thumbnailer/models/image'
+
+module LinkThumbnailer
+  module Scrapers
+    class Base < ::SimpleDelegator
+
+      attr_reader :config, :document, :website, :attribute_name
+
+      def initialize(document)
+        @config   = ::LinkThumbnailer.page.config
+        @document = document
+
+        super(config)
+      end
+
+      def call(website, attribute_name)
+        @website        = website
+        @attribute_name = attribute_name
+
+        website.send("#{attribute_name}=", value)
+        website
+      end
+
+      def applicable?
+        true
+      end
+
+      private
+
+      def value
+        raise 'must implement'
+      end
+
+      def meta_xpath(options = {})
+        meta_xpaths(options).first
+      end
+
+      def meta_xpaths(options = {})
+        key       = options.fetch(:key, :property)
+        value     = options.fetch(:value, :content)
+        attribute = options.fetch(:attribute, attribute_name)
+
+        document.xpath("//meta[translate(@#{key},'#{abc.upcase}','#{abc}') = '#{attribute}' and @#{value}]")
+      end
+
+      def abc
+        'abcdefghijklmnopqrstuvwxyz'
+      end
+
+      def model_class
+        "::LinkThumbnailer::Models::#{attribute_name.to_s.camelize}".constantize
+      end
+
+      def modelize(node, text = nil)
+        model_class.new(node, text)
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/default/base.rb

@@ -0,0 +1,10 @@
+require 'link_thumbnailer/scrapers/base'
+
+module LinkThumbnailer
+  module Scrapers
+    module Default
+      class Base < ::LinkThumbnailer::Scrapers::Base
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/default/description.rb

@@ -0,0 +1,47 @@
+require 'link_thumbnailer/scrapers/default/base'
+
+module LinkThumbnailer
+  module Scrapers
+    module Default
+      class Description < ::LinkThumbnailer::Scrapers::Default::Base
+
+        private
+
+        def value
+          return model_from_meta.to_s if model_from_meta
+          return model_from_body.to_s if model_from_body
+          nil
+        end
+
+        def model_from_meta
+          modelize(node_from_meta, node_from_meta.attributes['content'].value) if node_from_meta
+        end
+
+        def model_from_body
+          nodes_from_body.each_with_index.map { |node, i| modelize(node, node.text, i + 1) }.sort.last
+        end
+
+        def node_from_meta
+          @node_from_meta ||= meta_xpath(key: :name)
+        end
+
+        def nodes_from_body
+          candidates.select { |node| valid_paragraph?(node) }
+        end
+
+        def valid_paragraph?(node)
+          true
+        end
+
+        def candidates
+          document.css('p,td')
+        end
+
+        def modelize(node, text, i = 1)
+          model_class.new(node, text, i)
+        end
+
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/default/images.rb

@@ -0,0 +1,64 @@
+require 'link_thumbnailer/scrapers/default/base'
+require 'link_thumbnailer/models/image'
+
+module LinkThumbnailer
+  module Scrapers
+    module Default
+      class Images < ::LinkThumbnailer::Scrapers::Default::Base
+
+        private
+
+        def value
+          abs_urls.each_with_index.take_while { |_, i| i < config.image_limit }.map { |e| modelize(e.first) }
+        end
+
+        def urls
+          document.search('//img').map { |i| i['src'] }.compact
+        end
+
+        def abs_urls
+          urls.map do |url|
+            uri = validate_url(url)
+
+            next unless uri
+
+            uri = prefix_uri(uri) if needs_prefix?(uri)
+            uri
+          end
+        end
+
+        def validate_url(url)
+          URI(url)
+        rescue URI::InvalidURIError
+          nil
+        end
+
+        def needs_prefix?(uri)
+          !uri.is_a?(URI::HTTP)
+        end
+
+        def prefix_uri(uri)
+          URI.join(prefix_url, uri)
+        end
+
+        def prefix_url
+          base_href || website.url
+        end
+
+        def base_href
+          base = document.at('//head/base')
+          base['href'] if base
+        end
+
+        def model_class
+          ::LinkThumbnailer::Models::Image
+        end
+
+        def modelize(uri)
+          model_class.new(uri)
+        end
+
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/default/title.rb

@@ -0,0 +1,25 @@
+require 'link_thumbnailer/scrapers/default/base'
+
+module LinkThumbnailer
+  module Scrapers
+    module Default
+      class Title < ::LinkThumbnailer::Scrapers::Default::Base
+
+        private
+
+        def value
+          model.to_s
+        end
+
+        def model
+          modelize(node)
+        end
+
+        def node
+          document.css(attribute_name)
+        end
+
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/opengraph/base.rb

@@ -0,0 +1,43 @@
+require 'link_thumbnailer/scrapers/base'
+
+module LinkThumbnailer
+  module Scrapers
+    module Opengraph
+      class Base < ::LinkThumbnailer::Scrapers::Base
+
+        def applicable?
+          meta.any? { |node| opengraph_node?(node) }
+        end
+
+        private
+
+        def value
+          model.to_s
+        end
+
+        def model
+          modelize(node, node.attributes['content'].to_s) if node
+        end
+
+        def node
+          @node ||= meta_xpath(attribute: attribute) ||
+                      meta_xpath(attribute: attribute, key: :name)
+        end
+
+        def attribute
+          "og:#{attribute_name}"
+        end
+
+        def opengraph_node?(node)
+          node.attribute('name').to_s.start_with?('og:') ||
+            node.attribute('property').to_s.start_with?('og:')
+        end
+
+        def meta
+          document.css('meta')
+        end
+
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/opengraph/description.rb

@@ -0,0 +1,10 @@
+require 'link_thumbnailer/scrapers/opengraph/base'
+
+module LinkThumbnailer
+  module Scrapers
+    module Opengraph
+      class Description < ::LinkThumbnailer::Scrapers::Opengraph::Base
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/opengraph/image.rb

@@ -0,0 +1,30 @@
+require 'link_thumbnailer/scrapers/opengraph/base'
+
+module LinkThumbnailer
+  module Scrapers
+    module Opengraph
+      class Image < ::LinkThumbnailer::Scrapers::Opengraph::Base
+
+        private
+
+        def value
+          model
+        end
+
+        def model
+          nodes.map { |n| modelize(n, n.attributes['content'].to_s) }
+        end
+
+        def modelize(node, text = nil)
+          model_class.new(text)
+        end
+
+        def nodes
+          nodes = meta_xpaths(attribute: attribute)
+          nodes.empty? ? meta_xpaths(attribute: attribute, key: :name) : nodes
+        end
+
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/opengraph/images.rb

@@ -0,0 +1,16 @@
+require 'link_thumbnailer/scrapers/opengraph/base'
+require 'link_thumbnailer/scrapers/opengraph/image'
+
+module LinkThumbnailer
+  module Scrapers
+    module Opengraph
+      class Images < ::LinkThumbnailer::Scrapers::Opengraph::Base
+
+        def call(website, attribute_name)
+          ::LinkThumbnailer::Scrapers::Opengraph::Image.new(document).call(website, 'image')
+        end
+
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/scrapers/opengraph/title.rb

@@ -0,0 +1,10 @@
+require 'link_thumbnailer/scrapers/opengraph/base'
+
+module LinkThumbnailer
+  module Scrapers
+    module Opengraph
+      class Title < ::LinkThumbnailer::Scrapers::Opengraph::Base
+      end
+    end
+  end
+end

data/lib/link_thumbnailer/version.rb

@@ -1,3 +1,3 @@
-module LinkThumbnailer
-  VERSION = "1.1.2"
-end
+module LinkThumbnailer
+  VERSION = '2.0.0'
+end