busk-ruby-readability 1.1.1 → 1.2.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    busk-ruby-readability (1.1.0)
+    busk-ruby-readability (1.1.1)
 
 GEM
   remote: http://rubygems.org/

data/lib/readability.rb

@@ -42,6 +42,11 @@ module Readability
       @best_candidate = nil
     end
 
+    def has_special_rule?
+      !!rules[@base_uri]
+    end
+
+
     def content(remove_unlikely_candidates = true)
       debug "Starting the content heuristic"
       @document.css("script, style").each {|el| el.remove }
@@ -50,6 +55,7 @@ module Readability
       article = youtube if is_youtube? && remove_unlikely_candidates
       article = vimeo if is_vimeo? && remove_unlikely_candidates
       article = ted if is_ted? && remove_unlikely_candidates
+      article = apply_custom_rule if has_special_rule?
 
       if article && remove_unlikely_candidates
         return article.to_html.gsub(/[\r\n\f]+/, "\n" ).gsub(/[\t ]+/, " ").gsub(/ /, " ")
@@ -87,6 +93,10 @@ module Readability
       (@base_uri.to_s =~ REGEXES[:videoRe])
     end
 
+
+
+
+
     def youtube
       debug("I have a Youtube video page")
       if @request =~ /\?v=([_\-a-z0-9]+)&?/i
@@ -141,26 +151,28 @@ module Readability
 
       sibling_score_threshold = [10, best_candidate[:content_score] * 0.2].max
       output = Nokogiri::XML::Node.new('div', @document)
-      best_candidate[:elem].parent.try(:children).each do |sibling|
-        append = false
-        append = true if sibling == best_candidate[:elem]
-        append = true if candidates[sibling] && candidates[sibling][:content_score] >= sibling_score_threshold
-
-        if sibling.name.downcase == "p"
-          link_density = get_link_density(sibling)
-          node_content = sibling.text
-          node_length = node_content.length
-
-          if node_length > 80 && link_density < 0.25
-            append = true
-          elsif node_length < 80 && link_density == 0 && node_content =~ /\.( |$)/
-            append = true
+      begin
+        best_candidate[:elem].parent.try(:children).each do |sibling|
+          append = false
+          append = true if sibling == best_candidate[:elem]
+          append = true if candidates[sibling] && candidates[sibling][:content_score] >= sibling_score_threshold
+
+          if sibling.name.downcase == "p"
+            link_density = get_link_density(sibling)
+            node_content = sibling.text
+            node_length = node_content.length
+
+            if node_length > 80 && link_density < 0.25
+              append = true
+            elsif node_length < 80 && link_density == 0 && node_content =~ /\.( |$)/
+              append = true
+            end
           end
-        end
 
-        if append
-          sibling.name = "div" unless %w[div p].include?(sibling.name.downcase)
-          output << sibling
+          if append
+            sibling.name = "div" unless %w[div p].include?(sibling.name.downcase)
+            output << sibling
+          end
         end
       end
 
@@ -284,14 +296,7 @@ module Readability
             debug("Altering div(##{elem[:id]}.#{elem[:class]}) to p");
             elem.name = "p"
           end
-        else
-          # wrap text nodes in p tags
-#          elem.children.each do |child|
-#            if child.text?
-##              debug("wrapping text node with a p")
-#              child.swap("<p>#{child.text}</p>")
-#            end
-#          end
+
         end
       end
     end
@@ -387,5 +392,33 @@ module Readability
       node.to_html.gsub(/[\r\n\f]+/, "\n" ).gsub(/[\t ]+/, " ").gsub(/&nbsp;/, " ")
     end
 
+    private
+
+    def rules
+      @rules ||= YAML.load_file("special_rules.yml")["sites"]
+    end
+
+    def apply_custom_rule
+      extracted = @document.css(rules[@base_uri]["css"])
+      extracted.each do |elem|
+        if (elem.try(:inner_html) =~ /^\W*$/)
+          extracted.delete elem
+        end
+      end
+      extracted
+    end
+
   end
+
+
+  private
+
+  def remove_empty_tags(chunk)
+    chunk.css("p").each do |elem|
+      elem.remove if elem.content.strip.empty?
+    end
+  end
+
+
+
 end

data/ruby-readability.gemspec

@@ -2,7 +2,7 @@ Gem::Specification.new do |s|
   s.authors = ["Fabio Mont Alegre", "Rodrigo Flores"]
   s.email = "it-team@busk.com"
   s.homepage = "http://github.com/busk/ruby-readability"
-  s.version = "1.1.1"
+  s.version = "1.2.0"
   s.name = "busk-ruby-readability"
   s.summary = "A rewrite of original ruby-readability"
 

data/spec/fixtures/folha.html

@@ -1,3 +1,4 @@
+
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
 <html>
 <head>