link_thumbnailer 3.3.2

data/Rakefile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+
+task default: :spec
+task test:    :spec

data/lib/generators/link_thumbnailer/install_generator.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module LinkThumbnailer
+  module Generators
+    class InstallGenerator < ::Rails::Generators::Base
+
+      source_root File.expand_path('../../templates', __FILE__)
+
+      desc 'Creates a LinkThumbnailer initializer for your application.'
+
+      def copy_initializer
+        template 'initializer.rb', 'config/initializers/link_thumbnailer.rb'
+      end
+
+    end
+  end
+end

data/lib/generators/templates/initializer.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+# Use this hook to configure LinkThumbnailer bahaviors.
+LinkThumbnailer.configure do |config|
+  # Numbers of redirects before raising an exception when trying to parse given url.
+  #
+  # config.redirect_limit = 3
+
+  # Set user agent
+  #
+  # config.user_agent = 'link_thumbnailer'
+
+  # Enable or disable SSL verification
+  #
+  # config.verify_ssl = true
+
+  # The amount of time in seconds to wait for a connection to be opened.
+  # If the HTTP object cannot open a connection in this many seconds,
+  # it raises a Net::OpenTimeout exception.
+  #
+  # See http://www.ruby-doc.org/stdlib-2.1.1/libdoc/net/http/rdoc/Net/HTTP.html#open_timeout
+  #
+  # config.http_open_timeout = 5
+
+  # List of blacklisted urls you want to skip when searching for images.
+  #
+  # config.blacklist_urls = [
+  #   %r{^http://ad\.doubleclick\.net/},
+  #   %r{^http://b\.scorecardresearch\.com/},
+  #   %r{^http://pixel\.quantserve\.com/},
+  #   %r{^http://s7\.addthis\.com/}
+  # ]
+
+  # List of attributes you want LinkThumbnailer to fetch on a website.
+  #
+  # config.attributes = [:title, :images, :description, :videos, :favicon]
+
+  # List of procedures used to rate the website description. Add you custom class
+  # here. See wiki for more details on how to build your own graders.
+  #
+  # config.graders = [
+  #   ->(description) { ::LinkThumbnailer::Graders::Length.new(description) },
+  #   ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :class) },
+  #   ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :id) },
+  #   ->(description) { ::LinkThumbnailer::Graders::Position.new(description, weight: 3) },
+  #   ->(description) { ::LinkThumbnailer::Graders::LinkDensity.new(description) }
+  # ]
+
+  # Minimum description length for a website.
+  #
+  # config.description_min_length = 25
+
+  # Regex of words considered positive to rate website description.
+  #
+  # config.positive_regex = /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i
+
+  # Regex of words considered negative to rate website description.
+  #
+  # config.negative_regex = /combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget|modal/i
+
+  # Numbers of images to fetch. Fetching too many images will be slow.
+  # Note that LinkThumbnailer will only sort fetched images between each other.
+  # Meaning that they could be a "better" image on the page.
+  #
+  # config.image_limit = 5
+
+  # Whether you want LinkThumbnailer to return image size and type or not.
+  # Setting this value to false will increase performance since for each images, LinkThumbnailer
+  # does not have to fetch its size and type.
+  #
+  # config.image_stats = true
+
+  # Whether you want LinkThumbnailer to raise an exception if the Content-Type of the HTTP request
+  # is not an html or xml.
+  #
+  # config.raise_on_invalid_format = false
+
+  # Sets number of concurrent http connections that can be opened to fetch images informations such as size and type.
+  #
+  # config.max_concurrency = 20
+
+  # Defines the strategies to use to scrap the website. See the [Open Graph Protocol](http://ogp.me/) for more information.
+  #
+  # config.scrapers = [:opengraph, :default]
+
+  # Sets the default encoding.
+  #
+  # config.encoding = 'utf-8'
+end

data/lib/link_thumbnailer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'link_thumbnailer/version'
+require 'link_thumbnailer/configuration'
+require 'link_thumbnailer/exceptions'
+require 'link_thumbnailer/page'
+
+module LinkThumbnailer
+
+  class << self
+
+    attr_reader :page
+
+    def generate(url, options = {})
+      @page = ::LinkThumbnailer::Page.new(url, options)
+
+      page.generate
+    end
+
+  end
+
+end
+
+begin
+  require 'rails'
+rescue LoadError
+end
+
+$stderr.puts <<-EOC if !defined?(Rails)
+warning: no framework detected.
+
+Your Gemfile might not be configured properly.
+---- e.g. ----
+Rails:
+    gem 'link_thumbnailer'
+
+EOC

data/lib/link_thumbnailer/configuration.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module LinkThumbnailer
+
+  # Access point for the gem configurations.
+  #
+  # @return [LinkThumbnailer::Configuration] a configuration instance.
+  def self.config
+    @config ||= Configuration.new
+  end
+
+  # Configure hook used in the gem initializer. Convinient way to set all the
+  # gem configurations.
+  #
+  # @example inside config/initializers/link_thumbnaler.rb
+  #   LinkThumbnailer.configure do |config|
+  #     config.user_agent = 'link_thumbnailer'
+  #   end
+  #
+  # @return [void]
+  def self.configure
+    yield config if block_given?
+  end
+
+	class Configuration
+
+    attr_accessor :redirect_limit, :blacklist_urls, :user_agent,
+                  :verify_ssl, :http_open_timeout, :http_read_timeout, :attributes,
+                  :graders, :description_min_length, :positive_regex, :negative_regex,
+                  :image_limit, :image_stats, :raise_on_invalid_format, :max_concurrency,
+                  :scrapers, :http_override_headers, :encoding
+
+    alias_method :http_timeout, :http_open_timeout
+    alias_method :http_timeout=, :http_open_timeout=
+
+    # Create a new instance.
+    #
+    # @return [LinkThumbnailer::Configuration]
+    def initialize
+      @redirect_limit         = 3
+      @user_agent             = 'link_thumbnailer'
+      @verify_ssl             = true
+      @http_open_timeout      = 5
+      @http_read_timeout      = 5
+      @blacklist_urls         = [
+        %r{^http://ad\.doubleclick\.net/},
+        %r{^http://b\.scorecardresearch\.com/},
+        %r{^http://pixel\.quantserve\.com/},
+        %r{^http://s7\.addthis\.com/}
+      ]
+      @attributes             = [:title, :images, :description, :videos, :favicon]
+      @graders                = [
+        ->(description) { ::LinkThumbnailer::Graders::Length.new(description) },
+        ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :class) },
+        ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :id) },
+        ->(description) { ::LinkThumbnailer::Graders::Position.new(description, weigth: 3) },
+        ->(description) { ::LinkThumbnailer::Graders::LinkDensity.new(description) },
+      ]
+      @description_min_length = 50
+      @positive_regex = /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i
+      @negative_regex = /combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget|modal/i
+      @image_limit    = 5
+      @image_stats    = true
+      @raise_on_invalid_format = false
+      @max_concurrency = 20
+      @scrapers = [:opengraph, :default]
+      @http_override_headers = { 'Accept-Encoding' => 'none' }
+      @encoding = 'utf-8'
+    end
+
+	end
+end

data/lib/link_thumbnailer/exceptions.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module LinkThumbnailer
+  Exceptions         = Class.new(StandardError)
+  RedirectLimit      = Class.new(Exceptions)
+  BadUriFormat       = Class.new(Exceptions)
+  FormatNotSupported = Class.new(Exceptions)
+  ScraperInvalid     = Class.new(Exceptions)
+  HTTPError          = Class.new(Exceptions)
+  SyntaxError        = Class.new(Exceptions)
+end

data/lib/link_thumbnailer/grader.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'delegate'
+require 'link_thumbnailer/graders/base'
+require 'link_thumbnailer/graders/length'
+require 'link_thumbnailer/graders/html_attribute'
+require 'link_thumbnailer/graders/link_density'
+require 'link_thumbnailer/graders/position'
+
+module LinkThumbnailer
+  class Grader < ::SimpleDelegator
+
+    attr_reader :config, :description
+
+    def initialize(description)
+      @config      = ::LinkThumbnailer.page.config
+      @description = description
+
+      super(config)
+    end
+
+    # For given description, computes probabilities returned by each graders by multipying them together.
+    #
+    # @return [Float] the probability for the given description to be considered good
+    def call
+      probability = 1.0
+
+      graders.each do |lambda|
+        instance = lambda.call(description)
+        probability *= instance.call.to_f ** instance.weight
+      end
+
+      probability
+    end
+
+    private
+
+    def graders
+      config.graders
+    end
+
+  end
+end

data/lib/link_thumbnailer/graders/base.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'delegate'
+
+module LinkThumbnailer
+  module Graders
+    class Base < ::SimpleDelegator
+
+      attr_reader :config, :description, :options
+
+      def initialize(description, options = {})
+        @config      = ::LinkThumbnailer.page.config
+        @description = description
+        @options     = options
+
+        super(config)
+      end
+
+      def call
+        fail NotImplementedError
+      end
+
+      def weight
+        options.fetch(:weigth, 1)
+      end
+
+      private
+
+      def node
+        description.node
+      end
+
+      def text
+        description.text
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/graders/html_attribute.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module LinkThumbnailer
+  module Graders
+    class HtmlAttribute < ::LinkThumbnailer::Graders::Base
+
+      attr_reader :attribute_name
+
+      def initialize(description, attribute_name)
+        super(description)
+        @attribute_name = attribute_name.to_sym
+      end
+
+      def call
+        return 1.0 if positive?
+        return 0.0 if negative?
+        1.0
+      end
+
+      private
+
+      def attribute
+        node[attribute_name]
+      end
+
+      def attribute?
+        attribute && !attribute.empty?
+      end
+
+      def negative?
+        attribute? && attribute =~ negative_regex
+      end
+
+      def positive?
+        attribute? && attribute =~ positive_regex
+      end
+
+      def negative_regex
+        config.negative_regex
+      end
+
+      def positive_regex
+        config.positive_regex
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/graders/length.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module LinkThumbnailer
+  module Graders
+    class Length < ::LinkThumbnailer::Graders::Base
+
+      def call
+        return 0.0 if too_short?
+
+        y / get_gaussian_value_for(ideal_description_length)
+      end
+
+      private
+
+      def get_gaussian_value_for(x)
+        Math.sqrt(2.0 * Math::PI ** 2) * Math.exp(-(x - ideal_description_length) ** 2 / 2.0 * 0.005 ** 2)
+      end
+
+      def x
+        text.length
+      end
+
+      def y
+        get_gaussian_value_for(x)
+      end
+
+      def ideal_description_length
+        options.fetch(:ideal_description_length, 120).to_f
+      end
+
+      def too_short?
+        text.length < config.description_min_length
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/graders/link_density.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module LinkThumbnailer
+  module Graders
+    class LinkDensity < ::LinkThumbnailer::Graders::Base
+
+      def call
+        return 0.0 if text.length == 0
+        1.0 - (links.count.to_f / text.length.to_f)
+      end
+
+      private
+
+      def links
+        node.css('a').map(&:text).compact.reject(&:empty?)
+      end
+
+    end
+  end
+end

data/lib/link_thumbnailer/graders/position.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module LinkThumbnailer
+  module Graders
+    class Position < ::LinkThumbnailer::Graders::Base
+
+      def call
+        1.0 - (description.position.to_f / description.candidates_number.to_f)
+      end
+
+    end
+  end
+end