busk-ruby-readability 1.0.7 → 1.1.0

data/Gemfile

@@ -0,0 +1,10 @@
+source "http://rubygems.org"
+
+gem "nokogiri"
+
+group :test do
+  gem 'rspec'
+  gem 'fakeweb'
+end
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,28 @@
+PATH
+  remote: .
+  specs:
+    busk-ruby-readability (1.1.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    fakeweb (1.3.0)
+    nokogiri (1.4.4)
+    rspec (2.4.0)
+      rspec-core (~> 2.4.0)
+      rspec-expectations (~> 2.4.0)
+      rspec-mocks (~> 2.4.0)
+    rspec-core (2.4.0)
+    rspec-expectations (2.4.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  busk-ruby-readability!
+  fakeweb
+  nokogiri
+  rspec

data/README

@@ -0,0 +1,9 @@
+This code is under the Apache License 2.0.  http://www.apache.org/licenses/LICENSE-2.0
+
+This is a ruby port of arc90's readability project
+
+http://lab.arc90.com/experiments/readability/
+
+Given a html document, it pulls out the main body text and cleans it up.
+
+Ruby port by starrhorne and iterationlabs

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/lib/readability.rb

@@ -43,6 +43,7 @@ module Readability
     end
 
     def content(remove_unlikely_candidates = true)
+      debug "Starting the content heuristic"
       @document.css("script, style").each {|el| el.remove }
       @document.search('//comment()').each {|el| el.remove }
 
@@ -58,9 +59,9 @@ module Readability
         candidates = score_paragraphs(options[:min_text_length] || TEXT_LENGTH_THRESHOLD)
         best_candidate = select_best_candidate(candidates)
         article = get_article(candidates, best_candidate)
-      
+
         cleaned_article = sanitize(article, candidates, options)
-      
+
         if remove_unlikely_candidates && article.text.strip.length < (options[:retry_length] || RETRY_LENGTH)
           make_html
           content(false)
@@ -69,24 +70,25 @@ module Readability
         end
       end
     end
-    
+
     def is_youtube?
-      (@base_uri.to_s =~ /^http:\/\/(www\.)?youtube.com/)
+      (@base_uri.to_s =~ /^(www\.)?youtube.com/)
     end
-    
+
     def is_vimeo?
-      (@base_uri.to_s =~ /^http:\/\/(www.)?vimeo.com/)
+      (@base_uri.to_s =~ /^(www.)?vimeo.com/)
     end
 
     def is_ted?
-      (@base_uri.to_s =~ /^http:\/\/(www.)?ted.com\/talks/)
+      (@base_uri.to_s =~ /^(www.)?ted.com\/talks/)
     end
-    
+
     def is_special_case?
       (@base_uri.to_s =~ REGEXES[:videoRe])
     end
-    
+
     def youtube
+      debug("I have a Youtube video page")
       if @request =~ /\?v=([_\-a-z0-9]+)&?/i
         Nokogiri::HTML.fragment <<-HTML
           <object width="706" height="422">
@@ -100,8 +102,9 @@ module Readability
         nil
       end
     end
-    
+
     def vimeo
+      debug("I have a Vimeo video page")
       # matches non-channel or pages that used swfobject to print player
       if @document.css("#clip_id")
         Nokogiri::HTML.fragment("<iframe src=\"http://player.vimeo.com/video/#{@document.css("#clip_id").attr('value')}\" width=\"572\" height=\"322\" frameborder=\"0\"></iframe>")
@@ -118,8 +121,9 @@ module Readability
         nil
       end
     end
-    
+
     def ted
+      debug("I have a TED video page")
       if (player = @document.css(".copy_paste")).present?
         unless player.first.attr("value").blank?
           Nokogiri::HTML.fragment(player.first.attr("value").to_s)
@@ -130,7 +134,7 @@ module Readability
         nil
       end
     end
-        
+
     def get_article(candidates, best_candidate)
       # Now that we have the top candidate, look through its siblings for content that might also be related.
       # Things like preambles, content split by ads that we removed, etc.
@@ -164,7 +168,7 @@ module Readability
     end
 
     def select_best_candidate(candidates)
-      @best_candidate ||= begin 
+      @best_candidate ||= begin
         sorted_candidates = candidates.values.sort { |a, b| b[:content_score] <=> a[:content_score] }
 
         debug("Top 5 candidates:")
@@ -239,6 +243,10 @@ module Readability
       weight
     end
 
+    def convert_to_utf8(string)
+      string.unpack("C*").pack("U*")
+    end
+
     def score_node(elem)
       content_score = class_weight(elem)
       case elem.name.downcase
@@ -255,7 +263,7 @@ module Readability
     end
 
     def debug(str)
-      puts str if options[:debug]
+      puts "READABILITY : "+ str if options[:debug]
     end
 
     def remove_unlikely_candidates!
@@ -296,7 +304,7 @@ module Readability
       node.css("form").each do |elem|
         elem.remove
       end
-      
+
       node.css("iframe").each do |iframe|
         unless iframe.attr("src").to_s =~ REGEXES[:videoRe]
           iframe.remove
@@ -326,7 +334,7 @@ module Readability
           to_remove = false
           reason = ""
 
-          if (counts["img"] > counts["p"]) && (counts["p"] > 0) 
+          if (counts["img"] > counts["p"]) && (counts["p"] > 0)
             reason = "too many images #{counts['p']}"
             to_remove = true
           elsif counts["li"] > counts["p"] && name != "ul" && name != "ol"

data/ruby-readability.gemspec

@@ -0,0 +1,13 @@
+Gem::Specification.new do |s|
+  s.authors = ["Fabio Mont Alegre", "Rodrigo Flores"]
+  s.email = "it-team@busk.com"
+  s.homepage = "http://github.com/busk/ruby-readability"
+  s.version = "1.1.0"
+  s.name = "busk-ruby-readability"
+  s.summary = "A rewrite of original ruby-readability"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/spec/fixtures/folha.html

@@ -0,0 +1,29 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<title>Folha.com - Equil�brio e Sa�de - Anvisa detecta pelo de roedor e interdita ch� da Dr. Oetker - 27/01/2011</title>
+<meta name="title" content="Anvisa detecta pelo de roedor e interdita ch� da Dr. Oetker">
+<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
+</head>
+
+<body>
+<div id="articleBy">
+  <p>
+    COLABORA��O PARA A <b>FOLHA</b>
+  </p>
+
+  <p>
+    A Anvisa (Ag�ncia Nacional de Vigil�ncia Sanit�ria) interditou o lote do ch� de erva doce da marca Dr. Oetker. A medida foi     publicada no &quot;Di�rio Oficial da Uni�o&quot; na quarta-feira (26).
+  </p>
+
+  <p>
+    Segundo a Vigil�ncia Sanit�ria, o lote L160T02 do produto --data de validade 01/12/2011-- apresentou resultado insatisfat�rio no ensaio de pesquisa para mat�rias macrosc�picas e microsc�picas que detectou a presen�a de p�lo de roedor e fragmentos de inseto.
+  </p>
+
+  <p>
+    A interdi��o cautelar vale pelo per�odo de 90 dias ap�s a data de publica��o. Durante esse tempo, o produto interditado n�o deve ser consumido e nem comercializado. As pessoas que j� adquiriram o produto do lote suspenso devem interromper o  consumo.
+  </p>
+</div>
+</body>
+
+</html>

data/spec/readability_spec.rb

@@ -47,7 +47,7 @@ describe Readability do
           </body>
         </html>
       HTML
-      
+
       @doc = Readability::Document.new(@html, nil, nil)
       @elem1 = @doc.document.css("#elem1").first
       @elem2 = @doc.document.css("#elem2").first
@@ -105,7 +105,7 @@ describe Readability do
           </body>
         </html>
       HTML
-      
+
       @doc = Readability::Document.new(@html, nil, nil)
       @candidates = @doc.score_paragraphs(0)
     end
@@ -140,6 +140,27 @@ describe Readability do
     end
   end
 
+  describe 'dealing with iso-8859-1' do
+    before(:each) do
+      file = File.open('spec/fixtures/folha.html', 'r')
+      @content = file.read
+    end
+
+    it "should return the main page content" do
+      Readability::Document.new(Nokogiri::HTML(@content, nil, 'ISO-8859'),nil,nil).content.unpack("C*").pack("U*")  .should == "<div><div>\n <p>\n COLABORA\303\207\303\203O PARA A FOLHA\n </p>\n <p>\n A Anvisa (Ag\303\252ncia Nacional de Vigil\303\242ncia Sanit\303\241ria) interditou o lote do ch\303\241 de erva doce da marca Dr. Oetker. A medida foi publicada no \"Di\303\241rio Oficial da Uni\303\243o\" na quarta-feira (26).\n </p>\n <p>\n Segundo a Vigil\303\242ncia Sanit\303\241ria, o lote L160T02 do produto --data de validade 01/12/2011-- apresentou resultado insatisfat\303\263rio no ensaio de pesquisa para mat\303\251rias macrosc\303\263picas e microsc\303\263picas que detectou a presen\303\247a de p\303\252lo de roedor e fragmentos de inseto.\n </p>\n <p>\n A interdi\303\247\303\243o cautelar vale pelo per\303\255odo de 90 dias ap\303\263s a data de publica\303\247\303\243o. Durante esse tempo, o produto interditado n\303\243o deve ser consumido e nem comercializado. As pessoas que j\303\241 adquiriram o produto do lote suspenso devem interromper o consumo.\n </p>\n</div></div>"
+    end
+  end
+
+  describe 'dealing with utf-8' do
+    before do
+      @doc = Readability::Document.new(Nokogiri::HTML("<html><head><title>title!</title></head><body><div><p>Açougue, espátula, Vovô, çáóéãà</p></div></body>", nil, 'UTF-8'), nil, nil, :min_text_length => 0, :retry_length => 1)
+    end
+
+    it 'should return the main page content' do
+      @doc.content.should match("Açougue, espátula, Vovô, çáóéãà")
+    end
+  end
+
   describe "ignoring sidebars" do
     before do
       @doc = Readability::Document.new(Nokogiri::HTML("<html><head><title>title!</title></head><body><div><p>Some content</p></div><div class='sidebar'><p>sidebar<p></div></body>"), nil, nil, :min_text_length => 0, :retry_length => 1)
@@ -149,13 +170,13 @@ describe Readability do
       @doc.content.should_not match("sidebar")
     end
   end
-  
+
   describe "outputs good stuff for known documents" do
     before do
       @html_files = Dir.glob(File.dirname(__FILE__) + "/fixtures/samples/*.html")
       @samples = @html_files.map {|filename| File.basename(filename, '.html') }
     end
-    
+
     it "should output expected fragments of text" do
 
       checks = 0
@@ -165,12 +186,12 @@ describe Readability do
 
         load "fixtures/samples/#{sample}-fragments.rb"
         puts "testing #{sample}..."
-        
+
         $required_fragments.each do |required_text|
           doc.should include(required_text)
           checks += 1
         end
-        
+
         $excluded_fragments.each do |text_to_avoid|
           doc.should_not include(text_to_avoid)
           checks += 1
@@ -179,21 +200,20 @@ describe Readability do
       puts "Performed #{checks} checks."
     end
   end
-  
+
   describe "handles vimeo.com videos" do
-  
-    before(:each) do 
+
+    before(:each) do
       FakeWeb.register_uri(:get, 'http://vimeo.com/10365005',
                            :response => File.read("spec/fixtures/vimeo.com.html"))
       @uri = URI.parse("http://vimeo.com/10365005")
-                           
-      @content = Readability::Document.new(Nokogiri::HTML(open('http://vimeo.com/10365005')), @uri, @uri).content
+      @content = Readability::Document.new(Nokogiri::HTML(open('http://vimeo.com/10365005')), @uri.host, @uri.request_uri).content
     end
-    
+
     it "should extract the video from the page" do
-      @content.should include("<iframe src=\"http://player.vimeo.com/video/10365005")
+      @content.should include("<iframe src=\"http://player.vimeo.com/video/10365005\"")
     end
-  
+
   end
-  
+
 end

data/spec/spec_helper.rb

@@ -1,13 +1,8 @@
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'rubygems'
 require 'readability'
-require 'spec'
-require 'spec/autorun'
-require 'nokogiri'
-require 'open-uri'
-require 'fakeweb'
+require 'bundler'
 
-Spec::Runner.configure do |config|
+Bundler.setup(:test)
 
-end
+require 'open-uri'
+require 'fakeweb'

data/test_on_url.rb

@@ -0,0 +1,6 @@
+require 'rubygems'
+require 'open-uri'
+require 'lib/readability'
+
+text = open(ARGV.first).read
+p Readability::Document.new(text).content

metadata

@@ -1,26 +1,27 @@
 --- !ruby/object:Gem::Specification 
 name: busk-ruby-readability
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 19
   prerelease: false
   segments: 
   - 1
+  - 1
   - 0
-  - 7
-  version: 1.0.7
+  version: 1.1.0
 platform: ruby
-authors: []
-
+authors: 
+- Fabio Mont Alegre
+- Rodrigo Flores
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2011-01-05 00:00:00 -02:00
+date: 2011-01-27 00:00:00 -02:00
 default_executable: 
 dependencies: []
 
 description: 
-email: spiceee@gmail.com
+email: it-team@busk.com
 executables: []
 
 extensions: []
@@ -28,8 +29,14 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- Gemfile
+- Gemfile.lock
+- README
+- Rakefile
 - lib/readability.rb
+- ruby-readability.gemspec
 - spec/fixtures/cant_read.html
+- spec/fixtures/folha.html
 - spec/fixtures/sample.html
 - spec/fixtures/samples/channel4-1-fragments.rb
 - spec/fixtures/samples/channel4-1.html
@@ -40,6 +47,7 @@ files:
 - spec/readability_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
+- test_on_url.rb
 has_rdoc: true
 homepage: http://github.com/busk/ruby-readability
 licenses: []
@@ -49,8 +57,6 @@ rdoc_options: []
 
 require_paths: 
 - lib
-- spec
-- spec/fixtures
 required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
@@ -76,5 +82,16 @@ rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: A rewrite of original ruby-readability
-test_files: []
-
+test_files: 
+- spec/fixtures/cant_read.html
+- spec/fixtures/folha.html
+- spec/fixtures/sample.html
+- spec/fixtures/samples/channel4-1-fragments.rb
+- spec/fixtures/samples/channel4-1.html
+- spec/fixtures/samples/globemail-ottawa-cuts-fragments.rb
+- spec/fixtures/samples/globemail-ottawa-cuts.html
+- spec/fixtures/should_not_truncate.txt
+- spec/fixtures/vimeo.com.html
+- spec/readability_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb