link_thumbnailer 1.1.2 → 2.0.0

data/lib/generators/link_thumbnailer/install_generator.rb

@@ -6,10 +6,6 @@ module LinkThumbnailer
 
       desc 'Creates a LinkThumbnailer initializer for your application.'
 
-      def install
-        route "use_link_thumbnailer"
-      end
-
       def copy_initializer
         template 'initializer.rb', 'config/initializers/link_thumbnailer.rb'
       end

data/lib/generators/templates/initializer.rb

@@ -1,41 +1,63 @@
-# Use this hook to configure LinkThumbnailer bahaviors.
-LinkThumbnailer.configure do |config|
-  # Set mandatory attributes require for the website to be valid.
-  # You can set `strict` to false if you want to skip this validation.
-  # config.mandatory_attributes = %w(url title image)
-
-  # Whether you want to validate given website against mandatory attributes or not.
-  # config.strict = true
-
-  # Numbers of redirects before raising an exception when trying to parse given url.
-  # config.redirect_limit = 3
-
-  # List of blacklisted urls you want to skip when searching for images.
-  # config.blacklist_urls = [
-  #   %r{^http://ad\.doubleclick\.net/},
-  #   %r{^http://b\.scorecardresearch\.com/},
-  #   %r{^http://pixel\.quantserve\.com/},
-  #   %r{^http://s7\.addthis\.com/}
-  # ]
-
-  # Included Rmagick attributes for images. See http://www.imagemagick.org/RMagick/doc/
-  # for more details.
-  # 'source_url' is a custom attribute and should always be included since this
-  # is where you'll find the image url.
-  # config.image_attributes = %w(source_url size type)
-
-  # Fetch 10 images maximum.
-  # config.limit = 10
-
-  # Return top 5 images only.
-  # config.top = 5
-
-  # Set user agent
-  # config.user_agent = 'linkthumbnailer'
-
-  # Enable or disable SSL verification
-  # config.verify_ssl = true
-
-  # HTTP open_timeout: The amount of time in seconds to wait for a connection to be opened.
-  # config.http_timeout = 5
-end
+# Use this hook to configure LinkThumbnailer bahaviors.
+LinkThumbnailer.configure do |config|
+  # Numbers of redirects before raising an exception when trying to parse given url.
+  #
+  # config.redirect_limit = 3
+
+  # Set user agent
+  #
+  # config.user_agent = 'link_thumbnailer'
+
+  # Enable or disable SSL verification
+  #
+  # config.verify_ssl = true
+
+  # The amount of time in seconds to wait for a connection to be opened.
+  # If the HTTP object cannot open a connection in this many seconds,
+  # it raises a Net::OpenTimeout exception.
+  #
+  # See http://www.ruby-doc.org/stdlib-2.1.1/libdoc/net/http/rdoc/Net/HTTP.html#open_timeout
+  #
+  # config.http_timeout = 5
+
+  # List of blacklisted urls you want to skip when searching for images.
+  #
+  # config.blacklist_urls = [
+  #   %r{^http://ad\.doubleclick\.net/},
+  #   %r{^http://b\.scorecardresearch\.com/},
+  #   %r{^http://pixel\.quantserve\.com/},
+  #   %r{^http://s7\.addthis\.com/}
+  # ]
+
+  # List of attributes you want LinkThumbnailer to fetch on a website.
+  #
+  # config.attributes = [:title, :images, :description]
+
+  # List of procedures used to rate the website description. Add you custom class
+  # here. Note that the order matter to compute the score. See wiki for more details
+  # on how to build your own graders.
+  #
+  # config.graders = [
+  #   ->(description) { ::LinkThumbnailer::Graders::Length.new(description) },
+  #   ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :class) },
+  #   ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :id) },
+  #   ->(description) { ::LinkThumbnailer::Graders::Position.new(description) },
+  #   ->(description) { ::LinkThumbnailer::Graders::LinkDensity.new(description) }
+  # ]
+
+  # Minimum description length for a website.
+  #
+  # config.description_min_length = 25
+
+  # Regex of words considered positive to rate website description.
+  #
+  # config.positive_regex = /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i
+
+  # Regex of words considered negative to rate website description.
+  #
+  # config.negative_regex = /combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget|modal/i
+
+  # Numbers of images to fetch. Fetching too many images will be slow.
+  #
+  # config.image_limit = 5
+end

data/lib/link_thumbnailer/configuration.rb

@@ -1,16 +1,58 @@
-require 'hashie'
-
 module LinkThumbnailer
-	class Configuration < Hashie::Mash
 
-    def rmagick_attributes
-      LinkThumbnailer.logger.info "[DEPRECATION] rmagick_attributes is deprecated. Use image_attributes instead."
-      image_attributes
-    end
+  # Access point for the gem configurations.
+  #
+  # @return [LinkThumbnailer::Configuration] a configuration instance.
+  def self.config
+    @config ||= Configuration.new
+  end
+
+  # Configure hook used in the gem initializer. Convinient way to set all the
+  # gem configurations.
+  #
+  # @example inside config/initializers/link_thumbnaler.rb
+  #   LinkThumbnailer.configure do |config|
+  #     config.user_agent = 'link_thumbnailer'
+  #   end
+  #
+  # @return [void]
+  def self.configure
+    yield config if block_given?
+  end
+
+	class Configuration
+
+    attr_accessor :redirect_limit, :blacklist_urls, :user_agent,
+                  :verify_ssl, :http_timeout, :attributes, :graders,
+                  :description_min_length, :positive_regex, :negative_regex,
+                  :image_limit
 
-    def rmagick_attributes=(value)
-      LinkThumbnailer.logger.info "[DEPRECATION] rmagick_attributes is deprecated. Use image_attributes instead."
-      self.image_attributes = value
+    # Create a new instance.
+    #
+    # @return [LinkThumbnailer::Configuration]
+    def initialize
+      @redirect_limit         = 3
+      @user_agent             = 'link_thumbnailer'
+      @verify_ssl             = true
+      @http_timeout           = 5
+      @blacklist_urls         = [
+        %r{^http://ad\.doubleclick\.net/},
+        %r{^http://b\.scorecardresearch\.com/},
+        %r{^http://pixel\.quantserve\.com/},
+        %r{^http://s7\.addthis\.com/}
+      ]
+      @attributes             = [:title, :images, :description]
+      @graders                = [
+        ->(description) { ::LinkThumbnailer::Graders::Length.new(description) },
+        ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :class) },
+        ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :id) },
+        ->(description) { ::LinkThumbnailer::Graders::Position.new(description) },
+        ->(description) { ::LinkThumbnailer::Graders::LinkDensity.new(description) }
+      ]
+      @description_min_length = 25
+      @positive_regex = /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i
+      @negative_regex = /combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget|modal/i
+      @image_limit    = 5
     end
 
 	end

data/lib/link_thumbnailer/exceptions.rb

@@ -0,0 +1,6 @@
+module LinkThumbnailer
+  class Exceptions      < StandardError; end
+  class RedirectLimit   < Exceptions; end
+  class BadUriFormat    < Exceptions; end
+  class ScraperInvalid  < Exceptions; end
+end

data/lib/link_thumbnailer/grader.rb

@@ -0,0 +1,37 @@
+require 'delegate'
+require 'link_thumbnailer/graders/base'
+require 'link_thumbnailer/graders/length'
+require 'link_thumbnailer/graders/html_attribute'
+require 'link_thumbnailer/graders/link_density'
+require 'link_thumbnailer/graders/position'
+
+module LinkThumbnailer
+  class Grader < ::SimpleDelegator
+
+    attr_reader :config, :description
+
+    def initialize(description)
+      @config      = ::LinkThumbnailer.page.config
+      @description = description
+
+      super(config)
+    end
+
+    def call
+      score = 0
+      graders.each do |lambda|
+        instance = lambda.call(description)
+        score += instance.call(score)
+      end
+
+      score
+    end
+
+    private
+
+    def graders
+      config.graders
+    end
+
+  end
+end

data/lib/link_thumbnailer/graders/base.rb

@@ -0,0 +1,32 @@
+require 'delegate'
+
+module LinkThumbnailer
+  module Graders
+    class Base < ::SimpleDelegator
+
+      attr_reader :config, :description
+
+      def initialize(description)
+        @config      = ::LinkThumbnailer.page.config
+        @description = description
+
+        super(config)
+      end
+
+      def call(current_score)
+        raise 'must implement'
+      end
+
+      private
+
+      def node
+        description.node
+      end
+
+      def text
+        node.text
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/graders/html_attribute.rb

@@ -0,0 +1,49 @@
+module LinkThumbnailer
+  module Graders
+    class HtmlAttribute < ::LinkThumbnailer::Graders::Base
+
+      attr_reader :attribute_name
+
+      def initialize(description, attribute_name)
+        super(description)
+        @attribute_name = attribute_name.to_sym
+      end
+
+      def call(current_score)
+        return 0 unless attribute?
+
+        score = 0
+        score -= 25 if negative?
+        score += 25 if positive?
+        score
+      end
+
+      private
+
+      def attribute
+        node[attribute_name]
+      end
+
+      def attribute?
+        attribute && !attribute.empty?
+      end
+
+      def negative?
+        attribute =~ negative_regex
+      end
+
+      def positive?
+        attribute =~ positive_regex
+      end
+
+      def negative_regex
+        config.negative_regex
+      end
+
+      def positive_regex
+        config.positive_regex
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/graders/length.rb

@@ -0,0 +1,19 @@
+module LinkThumbnailer
+  module Graders
+    class Length < ::LinkThumbnailer::Graders::Base
+
+      def call(current_score)
+        return -Float::INFINITY if too_short?
+
+        [(text.length / 100).to_i, 3].min
+      end
+
+      private
+
+      def too_short?
+        text.length < config.description_min_length
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/graders/link_density.rb

@@ -0,0 +1,21 @@
+module LinkThumbnailer
+  module Graders
+    class LinkDensity < ::LinkThumbnailer::Graders::Base
+
+      def call(current_score)
+        current_score *= (1 - density)
+      end
+
+      private
+
+      def density
+        links.length / text.length.to_f
+      end
+
+      def links
+        node.css('a').map(&:text)
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/graders/position.rb

@@ -0,0 +1,11 @@
+module LinkThumbnailer
+  module Graders
+    class Position < ::LinkThumbnailer::Graders::Base
+
+      def call(current_score)
+        2.0 / description.position
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/image_comparator.rb

@@ -0,0 +1,24 @@
+require 'link_thumbnailer/image_comparators/base'
+require 'link_thumbnailer/image_comparators/size'
+
+module LinkThumbnailer
+  class ImageComparator
+
+    attr_reader :image
+
+    def initialize(image)
+      @image = image
+    end
+
+    def call(other)
+      size_comparator.call(other)
+    end
+
+    private
+
+    def size_comparator
+      ::LinkThumbnailer::ImageComparators::Size.new(image)
+    end
+
+  end
+end

data/lib/link_thumbnailer/image_comparators/base.rb

@@ -0,0 +1,17 @@
+module LinkThumbnailer
+  module ImageComparators
+    class Base
+
+      attr_reader :image
+
+      def initialize(image)
+        @image = image
+      end
+
+      def call
+        raise 'you must implement this method'
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/image_comparators/size.rb

@@ -0,0 +1,11 @@
+module LinkThumbnailer
+  module ImageComparators
+    class Size < ::LinkThumbnailer::ImageComparators::Base
+
+      def call(other)
+        (other.size.min ** 2) <=> (image.size.min ** 2)
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/image_parser.rb

@@ -0,0 +1,18 @@
+require 'link_thumbnailer/image_parsers/size'
+require 'link_thumbnailer/image_parsers/type'
+
+module LinkThumbnailer
+  class ImageParser
+    class << self
+
+      def size(image)
+        ::LinkThumbnailer::ImageParsers::Size.perform(image)
+      end
+
+      def type(image)
+        ::LinkThumbnailer::ImageParsers::Type.perform(image)
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/image_parsers/size.rb

@@ -0,0 +1,15 @@
+require 'fastimage'
+
+module LinkThumbnailer
+  module ImageParsers
+    class Size
+
+      def self.perform(image)
+        ::FastImage.size(image.src, raise_on_failure: true)
+      rescue ::FastImage::FastImageException
+        [0, 0]
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/image_parsers/type.rb

@@ -0,0 +1,15 @@
+require 'fastimage'
+
+module LinkThumbnailer
+  module ImageParsers
+    class Type
+
+      def self.perform(image)
+        ::FastImage.type(image.src, raise_on_failure: true)
+      rescue ::FastImage::FastImageException
+        :jpg
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/image_validator.rb

@@ -0,0 +1,30 @@
+require 'delegate'
+
+module LinkThumbnailer
+  class ImageValidator < ::SimpleDelegator
+
+    attr_reader :config, :image
+
+    def initialize(image)
+      @config = ::LinkThumbnailer.page.config
+      @image  = image
+
+      super(config)
+    end
+
+    def call
+      blacklist_urls.each do |url|
+        return false if image.src && image.src.to_s[url]
+      end
+
+      true
+    end
+
+    private
+
+    def blacklist_urls
+      config.blacklist_urls
+    end
+
+  end
+end

data/lib/link_thumbnailer/model.rb

@@ -0,0 +1,16 @@
+module LinkThumbnailer
+  class Model
+
+    def to_json(*args)
+      as_json.to_json(*args)
+    end
+
+    private
+
+    def sanitize(str)
+      return unless str
+      str.strip.gsub(/[\r\n\f]+/, "\n")
+    end
+
+  end
+end

data/lib/link_thumbnailer/models/description.rb

@@ -0,0 +1,34 @@
+require 'link_thumbnailer/model'
+require 'link_thumbnailer/grader'
+
+module LinkThumbnailer
+  module Models
+    class Description < ::LinkThumbnailer::Model
+
+      attr_reader   :node, :text, :position
+      attr_accessor :score
+
+      def initialize(node, text, position = 1)
+        @node     = node
+        @text     = sanitize(text)
+        @position = position
+        @score    = compute_score
+      end
+
+      def to_s
+        text
+      end
+
+      def <=>(other)
+        score <=> other.score
+      end
+
+      private
+
+      def compute_score
+        ::LinkThumbnailer::Grader.new(self).call
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/models/image.rb

@@ -0,0 +1,54 @@
+require 'link_thumbnailer/model'
+require 'link_thumbnailer/image_parser'
+require 'link_thumbnailer/image_comparator'
+require 'link_thumbnailer/image_validator'
+
+module LinkThumbnailer
+  module Models
+    class Image < ::LinkThumbnailer::Model
+
+      attr_reader :src, :type, :size
+
+      def initialize(src)
+        @src  = src
+        @size = parser.size(self)
+        @type = parser.type(self)
+      end
+
+      def to_s
+        src.to_s
+      end
+
+      def <=>(other)
+        comparator.call(other)
+      end
+
+      def valid?
+        validator.call
+      end
+
+      def as_json(*)
+        {
+          src:  src.to_s,
+          size: size,
+          type: type
+        }
+      end
+
+      private
+
+      def parser
+        ::LinkThumbnailer::ImageParser
+      end
+
+      def validator
+        ::LinkThumbnailer::ImageValidator.new(self)
+      end
+
+      def comparator
+        ::LinkThumbnailer::ImageComparator.new(self)
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/models/title.rb

@@ -0,0 +1,20 @@
+require 'link_thumbnailer/model'
+
+module LinkThumbnailer
+  module Models
+    class Title < ::LinkThumbnailer::Model
+
+      attr_reader :node, :text
+
+      def initialize(node, text = nil)
+        @node = node
+        @text = sanitize(text || node.text)
+      end
+
+      def to_s
+        text
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/models/website.rb

@@ -0,0 +1,39 @@
+require 'link_thumbnailer/model'
+
+module LinkThumbnailer
+  module Models
+    class Website < ::LinkThumbnailer::Model
+
+      attr_accessor :url, :title, :description, :images
+
+      def initialize
+        @images = []
+      end
+
+      def image=(image)
+        self.images = image
+      end
+
+      def images=(images)
+        Array(images).each do |image|
+          next unless image.valid?
+          @images << image
+        end
+      end
+
+      def images
+        @images.sort!
+      end
+
+      def as_json(*)
+        {
+          url:          url.to_s,
+          title:        title,
+          description:  description,
+          images:       images
+        }
+      end
+
+    end
+  end
+end