link_thumbnailer 2.6.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aa4bcf1a567d87a3fd739645cc72dea9c04badb1
-  data.tar.gz: b284daca927931ce058f024ec5425c0d3c2aadc2
+  metadata.gz: a6f72611485eeb14494ca2ecc6dc7c4a2c1d7723
+  data.tar.gz: 8968fef4fb3d981a568a6cd94b0de53cedd8833b
 SHA512:
-  metadata.gz: d5c617733e8aca11eef6604eaab3d077077bc7dfaa8f533dcb6297ebb466a9d7fa6dfa29b73d04711d818cb6cde4c1d7f813f4350699402283c5836f9ec5a05d
-  data.tar.gz: 841dd6b8ca5dc185a0e77b8931c6a5cfc9fe74340f35d528f80d7dbd1af3ede76933c6873d024c3b213361c5d94136469b5a6001f27b13e9c79917100f1f6f39
+  metadata.gz: 1aadd1793f33cec74721d013954627a552693d2a50fd1d1a7e630194b6d19fbe945c549797a9edb23bb2d1254c0b3dc77d6f737f41c32a4aa3c7bda3fea3f796
+  data.tar.gz: 65631f74a0d44eee946ce64cd3e4f55a1c0fbb55756f17855990c98efc5d161cf543dc34dc7f384414df1d78f469f1f39b8b1f30f15ad8c6b2a6ea07c2413003

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 3.0.0
+
+- Improved description sorting.
+- Refactored how graders work. More information [here](https://github.com/gottfrois/link_thumbnailer/wiki/How-to-build-your-own-Grader%3F)
+
 # 2.6.1
 
 - Fix remove useless dependency

data/lib/generators/templates/initializer.rb

@@ -68,4 +68,9 @@ LinkThumbnailer.configure do |config|
   # does not have to fetch its size and type.
   #
   # config.image_stats = true
+  #
+  # Whether you want LinkThumbnailer to raise an exception if the Content-Type of the HTTP request
+  # is not an html or xml.
+  #
+  # config.raise_on_invalid_format = false
 end

data/lib/link_thumbnailer/configuration.rb

@@ -51,9 +51,9 @@ module LinkThumbnailer
         ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :class) },
         ->(description) { ::LinkThumbnailer::Graders::HtmlAttribute.new(description, :id) },
         ->(description) { ::LinkThumbnailer::Graders::Position.new(description) },
-        ->(description) { ::LinkThumbnailer::Graders::LinkDensity.new(description) }
+        ->(description) { ::LinkThumbnailer::Graders::LinkDensity.new(description) },
       ]
-      @description_min_length = 25
+      @description_min_length = 50
       @positive_regex = /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i
       @negative_regex = /combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget|modal/i
       @image_limit    = 5

data/lib/link_thumbnailer/grader.rb

@@ -3,6 +3,7 @@ require 'link_thumbnailer/graders/base'
 require 'link_thumbnailer/graders/length'
 require 'link_thumbnailer/graders/html_attribute'
 require 'link_thumbnailer/graders/link_density'
+require 'link_thumbnailer/graders/punctuation_density'
 require 'link_thumbnailer/graders/position'
 
 module LinkThumbnailer
@@ -17,14 +18,18 @@ module LinkThumbnailer
       super(config)
     end
 
+    # For given description, computes probabilities returned by each graders by multipying them together.
+    #
+    # @return [Float] the probability for the given description to be considered good
     def call
-      score = 0
+      probability = 1.0
+
       graders.each do |lambda|
         instance = lambda.call(description)
-        score += instance.call(score)
+        probability *= instance.call.to_f
       end
 
-      score
+      probability
     end
 
     private

data/lib/link_thumbnailer/graders/base.rb

@@ -13,7 +13,7 @@ module LinkThumbnailer
         super(config)
       end
 
-      def call(current_score)
+      def call
         fail NotImplementedError
       end
 
@@ -24,7 +24,7 @@ module LinkThumbnailer
       end
 
       def text
-        node.text
+        description.text
       end
 
     end

data/lib/link_thumbnailer/graders/html_attribute.rb

@@ -9,13 +9,10 @@ module LinkThumbnailer
         @attribute_name = attribute_name.to_sym
       end
 
-      def call(current_score)
-        return 0 unless attribute?
-
-        score = 0
-        score -= 25 if negative?
-        score += 25 if positive?
-        score
+      def call
+        return 1.0 if positive?
+        return 0.0 if negative?
+        1.0
       end
 
       private
@@ -29,11 +26,11 @@ module LinkThumbnailer
       end
 
       def negative?
-        attribute =~ negative_regex
+        attribute? && attribute =~ negative_regex
       end
 
       def positive?
-        attribute =~ positive_regex
+        attribute? && attribute =~ positive_regex
       end
 
       def negative_regex

data/lib/link_thumbnailer/graders/length.rb

@@ -2,14 +2,34 @@ module LinkThumbnailer
   module Graders
     class Length < ::LinkThumbnailer::Graders::Base
 
-      def call(current_score)
-        return -Float::INFINITY if too_short?
+      def call
+        return 0.0 if too_short?
 
-        [(text.length / 100).to_i, 3].min
+        y / ideal_description_gaussian_value
       end
 
       private
 
+      def get_gaussian_value_for(x)
+        Math.sqrt(2.0 * Math::PI ** 2) * Math.exp(-(x - ideal_description_length) ** 2 / 2.0 * 0.005 ** 2)
+      end
+
+      def x
+        text.length
+      end
+
+      def y
+        get_gaussian_value_for(x)
+      end
+
+      def ideal_description_length
+        120.0
+      end
+
+      def ideal_description_gaussian_value
+        4.442882938158366
+      end
+
       def too_short?
         text.length < config.description_min_length
       end

data/lib/link_thumbnailer/graders/link_density.rb

@@ -2,22 +2,13 @@ module LinkThumbnailer
   module Graders
     class LinkDensity < ::LinkThumbnailer::Graders::Base
 
-      def call(current_score)
-        return 0 if density_ratio == 0
-        current_score *= density_ratio
+      def call
+        return 0.0 if text.length == 0
+        1.0 - (links.count.to_f / text.length.to_f)
       end
 
       private
 
-      def density
-        return 0 if text.length == 0
-        links.length / text.length.to_f
-      end
-
-      def density_ratio
-        1 - density
-      end
-
       def links
         node.css('a').map(&:text).compact.reject(&:empty?)
       end

data/lib/link_thumbnailer/graders/position.rb

@@ -2,8 +2,8 @@ module LinkThumbnailer
   module Graders
     class Position < ::LinkThumbnailer::Graders::Base
 
-      def call(current_score)
-        2.0 / description.position
+      def call
+        1.0 - (description.position.to_f / description.candidates_number.to_f)
       end
 
     end

data/lib/link_thumbnailer/models/description.rb

@@ -5,14 +5,15 @@ module LinkThumbnailer
   module Models
     class Description < ::LinkThumbnailer::Model
 
-      attr_reader   :node, :text, :position
-      attr_accessor :score
-
-      def initialize(node, text, position = 1)
-        @node     = node
-        @text     = sanitize(text)
-        @position = position
-        @score    = compute_score
+      attr_reader   :node, :text, :position, :candidates_number
+      attr_accessor :probability
+
+      def initialize(node, text, position = 1, candidates_number = 1)
+        @node              = node
+        @text              = sanitize(text)
+        @position          = position
+        @candidates_number = candidates_number
+        @probability       = compute_probability
       end
 
       def to_s
@@ -20,12 +21,12 @@ module LinkThumbnailer
       end
 
       def <=>(other)
-        score <=> other.score
+        probability <=> other.probability
       end
 
       private
 
-      def compute_score
+      def compute_probability
         ::LinkThumbnailer::Grader.new(self).call
       end
 

data/lib/link_thumbnailer/scrapers/default/description.rb

@@ -18,7 +18,7 @@ module LinkThumbnailer
         end
 
         def model_from_body
-          nodes_from_body.each_with_index.map { |node, i| modelize(node, node.text, i + 1) }.sort.last
+          nodes_from_body.each_with_index.map { |node, i| modelize(node, node.text, i) }.sort.last
         end
 
         def node_from_meta
@@ -37,8 +37,8 @@ module LinkThumbnailer
           document.css('p,td')
         end
 
-        def modelize(node, text, i = 1)
-          model_class.new(node, text, i)
+        def modelize(node, text, i = 0)
+          model_class.new(node, text, i, nodes_from_body.count)
         end
 
       end

data/lib/link_thumbnailer/version.rb

@@ -1,3 +1,3 @@
 module LinkThumbnailer
-  VERSION = '2.6.1'
+  VERSION = '3.0.0'
 end

data/spec/configuration_spec.rb

@@ -12,7 +12,7 @@ describe LinkThumbnailer::Configuration do
   it { expect(instance.blacklist_urls).to_not     be_empty }
   it { expect(instance.attributes).to             eq([:title, :images, :description, :videos, :favicon]) }
   it { expect(instance.graders).to_not            be_empty }
-  it { expect(instance.description_min_length).to eq(25) }
+  it { expect(instance.description_min_length).to eq(50) }
   it { expect(instance.positive_regex).to_not     be_nil }
   it { expect(instance.negative_regex).to_not     be_nil }
   it { expect(instance.image_limit).to            eq(5) }

data/spec/grader_spec.rb

@@ -7,8 +7,8 @@ describe LinkThumbnailer::Grader do
 
   describe '#call' do
 
-    let(:score)   { 10 }
-    let(:grader)  { double('grader', call: score) }
+    let(:probability)   { 0.5 }
+    let(:grader)  { double('grader', call: probability) }
     let(:lambda)  { ->(_) { grader } }
     let(:graders) { [lambda, lambda] }
     let(:action)  { instance.call }
@@ -17,7 +17,7 @@ describe LinkThumbnailer::Grader do
       instance.stub(:graders).and_return(graders)
     end
 
-    it { expect(action).to eq(score * graders.size) }
+    it { expect(action).to eq(0.5 * 0.5) }
 
   end
 

data/spec/graders/base_spec.rb

@@ -2,11 +2,11 @@ require 'spec_helper'
 
 describe LinkThumbnailer::Graders::Base do
 
-  let(:description) { double('description', node: node) }
+  let(:description) { double('description', text: 'foo', node: node) }
   let(:node)        { double('node', text: 'foo') }
   let(:instance)    { described_class.new(description) }
 
   it { expect(instance.send(:node)).to eq(description.node) }
-  it { expect(instance.send(:text)).to eq(description.node.text) }
+  it { expect(instance.send(:text)).to eq(description.text) }
 
 end

data/spec/graders/html_attribute_spec.rb

@@ -7,7 +7,7 @@ describe LinkThumbnailer::Graders::HtmlAttribute do
 
   describe '#call' do
 
-    let(:action) { instance.call(0) }
+    let(:action) { instance.call }
 
     context 'when current node does not match attribute' do
 
@@ -15,7 +15,7 @@ describe LinkThumbnailer::Graders::HtmlAttribute do
         instance.stub(:attribute?).and_return(false)
       end
 
-      it { expect(action).to eq(0) }
+      it { expect(action).to eq(1.0) }
 
     end
 
@@ -27,7 +27,7 @@ describe LinkThumbnailer::Graders::HtmlAttribute do
         instance.stub(:positive?).and_return(false)
       end
 
-      it { expect(action).to eq(-25) }
+      it { expect(action).to eq(0.0) }
 
     end
 
@@ -39,7 +39,7 @@ describe LinkThumbnailer::Graders::HtmlAttribute do
         instance.stub(:positive?).and_return(true)
       end
 
-      it { expect(action).to eq(25) }
+      it { expect(action).to eq(1.0) }
 
     end
 

data/spec/graders/length_spec.rb

@@ -12,7 +12,7 @@ describe LinkThumbnailer::Graders::Length do
 
   describe '#call' do
 
-    let(:action) { instance.call(0) }
+    let(:action) { instance.call }
 
     context 'when text is too short' do
 
@@ -20,7 +20,7 @@ describe LinkThumbnailer::Graders::Length do
         instance.stub(:too_short?).and_return(true)
       end
 
-      it { expect(action).to eq(-Float::INFINITY) }
+      it { expect(action).to eq(0.0) }
 
     end
 
@@ -31,19 +31,27 @@ describe LinkThumbnailer::Graders::Length do
         instance.stub(:text).and_return(text)
       end
 
-      context 'when text length is greater than 400' do
+      context 'when text length is 80' do
 
-        let(:text) { 'f' * 400 }
+        let(:text) { 'f' * 80 }
 
-        it { expect(action).to eq(3) }
+        it { expect(action).to eq(1.0) }
 
       end
 
-      context 'when text length is less than 300' do
+      context 'when text length is 100' do
 
-        let(:text) { 'f' * 299 }
+        let(:text) { 'f' * 100 }
 
-        it { expect(action).to eq(2) }
+        it { expect(action).to be < 1.0 }
+
+      end
+
+      context 'when text length is 60' do
+
+        let(:text) { 'f' * 60 }
+
+        it { expect(action).to be < 1.0 }
 
       end
 

data/spec/graders/link_density_spec.rb

@@ -7,54 +7,41 @@ describe LinkThumbnailer::Graders::LinkDensity do
 
   describe '#call' do
 
-    let(:previous_score)  { 1 }
-    let(:action)          { instance.call(previous_score) }
-
-    context 'when density_ratio is 0' do
-
-      before do
-        instance.stub(:density_ratio).and_return(0)
-      end
-
-      it { expect(action).to eq(0) }
+    let(:action) { instance.call }
 
+    before do
+      instance.stub(:text).and_return(text)
+      instance.stub(:links).and_return(links)
     end
 
-    context 'when density is not 0' do
+    context 'when text length is 0' do
 
-      before do
-        instance.stub(:density_ratio).and_return(10)
-      end
+      let(:text) { '' }
+      let(:links) { [] }
 
-      it { expect(action).to eq(10) }
+      it { expect(action).to eq(0.0) }
 
     end
 
-  end
+    context 'when text length is > 0' do
 
-  describe '#density' do
+      let(:text) { 'foo' }
 
-    let(:links)   { ['foo'] }
-    let(:action)  { instance.send(:density) }
+      context 'and links is 0' do
 
-    before do
-      instance.stub(:links).and_return(links)
-      instance.stub(:text).and_return(text)
-    end
+        let(:links) { [] }
 
-    context 'with text' do
+        it { expect(action).to eq(1.0) }
 
-      let(:text)    { 'abcd' }
-
-      it { expect(action).to eq(0.25) }
+      end
 
-    end
+      context 'and links is > 0' do
 
-    context 'without text' do
+        let(:links) { [1] }
 
-      let(:text)    { '' }
+        it { expect(action).to be_within(0.001).of(0.666) }
 
-      it { expect(action).to eq(0) }
+      end
 
     end
 

data/spec/graders/position_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe LinkThumbnailer::Graders::Position do
+
+  let(:description) { double('description') }
+  let(:instance)    { described_class.new(description) }
+
+  describe '#call' do
+
+    let(:action) { instance.call }
+
+    context 'when position is 0' do
+
+      before do
+        description.stub(:position).and_return(0)
+        description.stub(:candidates_number).and_return(1)
+      end
+
+      it { expect(action).to eq(1.0) }
+
+    end
+
+    context 'when position is 1 over 1 candidates' do
+
+      before do
+        description.stub(:position).and_return(1)
+        description.stub(:candidates_number).and_return(1)
+      end
+
+      it { expect(action).to eq(0.0) }
+
+    end
+
+    context 'when position is 1 over more than 1 candidates' do
+
+      before do
+        description.stub(:position).and_return(1)
+        description.stub(:candidates_number).and_return(2)
+      end
+
+      it { expect(action).to eq(0.5) }
+
+    end
+
+  end
+
+end

data/spec/graders/punctuation_density_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe LinkThumbnailer::Graders::PunctuationDensity do
+
+  let(:description) { double('description') }
+  let(:instance)    { described_class.new(description) }
+
+  describe '#call' do
+
+    let(:action) { instance.call }
+
+    before do
+      instance.stub(:text).and_return(text)
+      instance.stub(:punctuations).and_return(punctuations)
+    end
+
+    context 'when text length is 0' do
+
+      let(:text) { '' }
+      let(:punctuations) { [] }
+
+      it { expect(action).to eq(0.0) }
+
+    end
+
+    context 'when text length is > 0' do
+
+      let(:text) { 'foo' }
+
+      context 'and punctuations is 0' do
+
+        let(:punctuations) { [] }
+
+        it { expect(action).to eq(1.0) }
+
+      end
+
+      context 'and punctuations is > 0' do
+
+        let(:punctuations) { [1] }
+
+        it { expect(action).to be_within(0.001).of(0.666) }
+
+      end
+
+    end
+
+  end
+
+end

data/spec/models/description_spec.rb

@@ -22,37 +22,37 @@ describe LinkThumbnailer::Models::Description do
   describe '#<=>' do
 
     let(:another_instance)  { described_class.new(node, text) }
-    let(:score)             { 5 }
+    let(:probability)       { 0.5 }
     let(:action)            { instance <=> another_instance }
 
     before do
-      another_instance.score = score
+      another_instance.probability = probability
     end
 
-    context 'when instance score is lower' do
+    context 'when instance probability is lower' do
 
       before do
-        instance.score = score - 1
+        instance.probability = probability - 0.5
       end
 
       it { expect(action).to eq(-1) }
 
     end
 
-    context 'when instance score is equal' do
+    context 'when instance probability is equal' do
 
       before do
-        instance.score = score
+        instance.probability = probability
       end
 
       it { expect(action).to eq(0) }
 
     end
 
-    context 'when instance score is greater' do
+    context 'when instance probability is greater' do
 
       before do
-        instance.score = score + 1
+        instance.probability = probability + 0.5
       end
 
       it { expect(action).to eq(1) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: link_thumbnailer
 version: !ruby/object:Gem::Version
-  version: 2.6.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Pierre-Louis Gottfrois
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-22 00:00:00.000000000 Z
+date: 2015-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -193,6 +193,8 @@ files:
 - spec/graders/html_attribute_spec.rb
 - spec/graders/length_spec.rb
 - spec/graders/link_density_spec.rb
+- spec/graders/position_spec.rb
+- spec/graders/punctuation_density_spec.rb
 - spec/image_comparators/size_spec.rb
 - spec/image_parsers/size_spec.rb
 - spec/image_parsers/type_spec.rb
@@ -251,6 +253,8 @@ test_files:
 - spec/graders/html_attribute_spec.rb
 - spec/graders/length_spec.rb
 - spec/graders/link_density_spec.rb
+- spec/graders/position_spec.rb
+- spec/graders/punctuation_density_spec.rb
 - spec/image_comparators/size_spec.rb
 - spec/image_parsers/size_spec.rb
 - spec/image_parsers/type_spec.rb