ruby-readability 0.2.3 → 0.3.0.pre

data/.gitignore

@@ -0,0 +1,7 @@
+.DS_Store
+.gem
+.bundle
+Gemfile.lock
+pkg/*
+.idea
+.rvmrc

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in ruby-readability.gemspec
+gemspec

data/README

@@ -15,6 +15,30 @@ Example:
   source = open('http://lab.arc90.com/experiments/readability/').read
   puts Readability::Document.new(source).content
 
+Options:
+
+  You may provide additions options to Readability::Document.new, including:
+
+    :tags               - the base whitelist of tags to sanitize, defaults to %w[div p]
+    :remove_empty_nodes - remove <p> tags that have no text content; this will also remove p tags that contain only images
+    :attributes         - whitelist of allowed attributes
+    :debug              - provide debugging output, defaults false
+    :encoding           - if this page is of a known encoding, you can specify it; if left unspecified, the encoding will be guessed (only in Ruby 1.9.x)
+    :html_headers       - in Ruby 1.9.x these will be passed to the guess_html_encoding gem to aid with guessing the HTML encoding 
+
+Readability comes with a command-line tool for experimentation in bin/readability.
+
+  Usage: readability [options] URL
+      -d, --debug                      Show debug output
+      -i, --images                     Keep images and links
+      -h, --help                       Show this message
+
+Potential issues:
+
+* If you're on a Mac and are getting segmentation faults, see this discussion https://github.com/tenderlove/nokogiri/issues/404 and consider updating your version of libxml2.
+  Version 2.7.8 of libxml2 with the following worked for me:
+  gem install nokogiri -- --with-xml2-include=/usr/local/Cellar/libxml2/2.7.8/include/libxml2 --with-xml2-lib=/usr/local/Cellar/libxml2/2.7.8/lib --with-xslt-dir=/usr/local/Cellar/libxslt/1.1.26
+
 ===
 
 This code is under the Apache License 2.0.  http://www.apache.org/licenses/LICENSE-2.0

data/Rakefile

@@ -1,45 +1,6 @@
-require 'rubygems'
-require 'rake'
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "ruby-readability"
-    gem.summary = %Q{Port of arc90's readability project to ruby}
-    gem.description = %Q{Port of arc90's readability project to ruby}
-    gem.email = "andrew@iterationlabs.com"
-    gem.homepage = "http://github.com/iterationlabs/ruby-readability"
-    gem.authors = ["Andrew Cantino", "starrhorne", "libc", "Kyle Maxwell"]
-    gem.add_development_dependency "rspec", ">= 1.2.9"
-    gem.add_dependency 'nokogiri', '>= 1.4.2'
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
-end
-
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new(:spec) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.spec_files = FileList['spec/**/*_spec.rb']
-end
-
-Spec::Rake::SpecTask.new(:rcov) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
-end
-
-task :spec => :check_dependencies
+RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "ruby-readability #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end

data/bin/readability

@@ -2,12 +2,39 @@
 $KCODE='u'
 require 'rubygems'
 require 'open-uri'
+require 'optparse'
 require File.dirname(__FILE__) + '/../lib/readability'
 
+options = { :debug => false, :images => false }
+options_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename($0)} [options] URL"
+
+  opts.on("-d", "--debug", "Show debug output") do |v|
+    options[:debug] = v
+  end
+
+  opts.on("-i", "--images", "Keep images and links") do |i|
+    options[:images] = i
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end
+options_parser.parse!
+
 if ARGV.length != 1
-  STDERR.puts "Usage: #{File.basename($0)} URL"
+  STDERR.puts options_parser
   exit 1
 end
 
 text = open(ARGV.first).read
-puts Readability::Document.new(text).content
+if options[:images]
+  puts Readability::Document.new(text, :tags => %w[div p img a], 
+                                       :attributes => %w[src href], 
+                                       :remove_empty_nodes => false, 
+                                       :debug => options[:debug]).content
+else
+  puts Readability::Document.new(text, :debug => options[:debug]).content
+end

data/lib/readability.rb

@@ -1,5 +1,6 @@
 require 'rubygems'
 require 'nokogiri'
+require 'guess_html_encoding'
 
 module Readability
   class Document
@@ -8,14 +9,22 @@ module Readability
       :min_text_length => 25,
       :remove_unlikely_candidates => true,
       :weight_classes => true,
-      :clean_conditionally => true
+      :clean_conditionally => true,
+      :remove_empty_nodes => true
     }.freeze
 
     attr_accessor :options, :html
 
     def initialize(input, options = {})
-      @input = input.gsub(REGEXES[:replaceBrsRe], '</p><p>').gsub(REGEXES[:replaceFontsRe], '<\1span>')
       @options = DEFAULT_OPTIONS.merge(options)
+      @input = input
+
+      if RUBY_VERSION =~ /^1\.9\./ && !@options[:encoding]
+        @input = GuessHtmlEncoding.encode(@input, @options[:html_headers]) unless @options[:do_not_guess_encoding]
+        @options[:encoding] = @input.encoding.to_s
+      end
+
+      @input = @input.gsub(REGEXES[:replaceBrsRe], '</p><p>').gsub(REGEXES[:replaceFontsRe], '<\1span>')
       @remove_unlikely_candidates = @options[:remove_unlikely_candidates]
       @weight_classes = @options[:weight_classes]
       @clean_conditionally = @options[:clean_conditionally]
@@ -23,7 +32,7 @@ module Readability
     end
 
     def make_html
-      @html = Nokogiri::HTML(@input, nil, 'UTF-8')
+      @html = Nokogiri::HTML(@input, nil, @options[:encoding])
     end
 
     REGEXES = {
@@ -221,7 +230,7 @@ module Readability
           # wrap text nodes in p tags
 #          elem.children.each do |child|
 #            if child.text?
-##              debug("wrapping text node with a p")
+#              debug("wrapping text node with a p")
 #              child.swap("<p>#{child.text}</p>")
 #            end
 #          end
@@ -238,9 +247,11 @@ module Readability
         elem.remove
       end
 
-      # remove empty <p> tags
-      node.css("p").each do |elem|
-        elem.remove if elem.content.strip.empty?
+      if @options[:remove_empty_nodes]
+        # remove <p> tags that have no text content - this will also remove p tags that contain only images.
+        node.css("p").each do |elem|
+          elem.remove if elem.content.strip.empty?
+        end
       end
 
       # Conditionally clean <table>s, <ul>s, and <div>s
@@ -259,7 +270,6 @@ module Readability
       base_replace_with_whitespace.each { |tag| replace_with_whitespace[tag] = true }
 
       ([node] + node.css("*")).each do |el|
-
         # If element is in whitelist, delete all its attributes
         if whitelist[el.node_name]
           el.attributes.each { |a, x| el.delete(a) unless @options[:attributes] && @options[:attributes].include?(a.to_s) }
@@ -267,10 +277,9 @@ module Readability
           # Otherwise, replace the element with its contents
         else
           if replace_with_whitespace[el.node_name]
-            # Adding &nbsp; here, because swap removes regular spaaces
-            el.swap('&nbsp;' << el.text << '&nbsp;')
+            el.swap(Nokogiri::XML::Text.new(' ' << el.text << ' ', el.document))
           else
-            el.swap(el.text)
+            el.swap(Nokogiri::XML::Text.new(el.text, el.document))
           end
         end
 

data/lib/ruby-readability.rb

@@ -0,0 +1 @@
+require 'readability'

data/ruby-readability.gemspec

@@ -1,72 +1,24 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
-  s.name = %q{ruby-readability}
-  s.version = "0.2.3"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Andrew Cantino", "starrhorne", "libc", "Kyle Maxwell"]
-  s.date = %q{2011-01-26}
-  s.default_executable = %q{readability}
+  s.name        = "ruby-readability"
+  s.version     = '0.3.0.pre'
+  s.authors     = ["Andrew Cantino", "starrhorne", "libc", "Kyle Maxwell"]
+  s.email       = ["andrew@iterationlabs.com"]
+  s.homepage    = "http://github.com/iterationlabs/ruby-readability"
+  s.summary     = %q{Port of arc90's readability project to ruby}
   s.description = %q{Port of arc90's readability project to ruby}
-  s.email = %q{andrew@iterationlabs.com}
-  s.executables = ["readability"]
-  s.extra_rdoc_files = [
-    "README"
-  ]
-  s.files = [
-    ".document",
-    "README",
-    "Rakefile",
-    "VERSION",
-    "bin/readability",
-    "lib/readability.rb",
-    "ruby-readability.gemspec",
-    "spec/fixtures/cant_read.html",
-    "spec/fixtures/sample.html",
-    "spec/fixtures/samples/blogpost_with_links-fragments.rb",
-    "spec/fixtures/samples/blogpost_with_links.html",
-    "spec/fixtures/samples/channel4-1-fragments.rb",
-    "spec/fixtures/samples/channel4-1.html",
-    "spec/fixtures/samples/foxnews-india1-fragments.rb",
-    "spec/fixtures/samples/foxnews-india1.html",
-    "spec/fixtures/samples/globemail-ottawa-cuts-fragments.rb",
-    "spec/fixtures/samples/globemail-ottawa-cuts.html",
-    "spec/fixtures/should_not_truncate.txt",
-    "spec/readability_spec.rb",
-    "spec/spec.opts",
-    "spec/spec_helper.rb"
-  ]
-  s.homepage = %q{http://github.com/iterationlabs/ruby-readability}
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Port of arc90's readability project to ruby}
-  s.test_files = [
-    "spec/fixtures/samples/blogpost_with_links-fragments.rb",
-    "spec/fixtures/samples/channel4-1-fragments.rb",
-    "spec/fixtures/samples/foxnews-india1-fragments.rb",
-    "spec/fixtures/samples/globemail-ottawa-cuts-fragments.rb",
-    "spec/readability_spec.rb",
-    "spec/spec_helper.rb"
-  ]
 
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
+  s.rubyforge_project = "ruby-readability"
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
-      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.2"])
-    else
-      s.add_dependency(%q<rspec>, [">= 1.2.9"])
-      s.add_dependency(%q<nokogiri>, [">= 1.4.2"])
-    end
-  else
-    s.add_dependency(%q<rspec>, [">= 1.2.9"])
-    s.add_dependency(%q<nokogiri>, [">= 1.4.2"])
-  end
-end
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
 
+  s.add_development_dependency "rspec", ">= 2.6"
+  s.add_development_dependency "rr", ">= 1.0"
+  s.add_dependency 'nokogiri', '>= 1.4.2'
+  s.add_dependency 'guess_html_encoding', '>= 0.0.2'
+end

data/spec/fixtures/samples/blogpost_with_links-fragments.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 # This sample originally from http://softarhive.net
 
 $required_fragments = [

data/spec/fixtures/samples/channel4-1-fragments.rb

@@ -1,4 +1,4 @@
-
+# encoding: UTF-8
 # This sample originally from http://www.channel4.com/news/articles/world/judge+rules+briton+can+be+forcefed/3578372
 
 $required_fragments = [

data/spec/fixtures/samples/foxnews-india1-fragments.rb

@@ -1,4 +1,4 @@
-
+# encoding: UTF-8
 # This sample originally from http://www.foxnews.com/world/2010/05/14/police-killed-bus-touches-high-voltage-wire-central-india/?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed:+foxnews/latest+(Text+-+Latest+Headlines)
 
 $required_fragments = [

data/spec/fixtures/samples/globemail-ottawa-cuts-fragments.rb

@@ -1,4 +1,4 @@
-
+# encoding: UTF-8
 # Originally from http://www.theglobeandmail.com/news/national/ottawa-cuts-already-vacant-positions/article1494400/
 
 $required_fragments = [

data/spec/readability_spec.rb

@@ -1,4 +1,6 @@
-require File.expand_path(File.join(File.dirname(__FILE__), "spec_helper"))
+# encoding: UTF-8
+
+require 'spec_helper'
 
 describe Readability do
   before do
@@ -115,11 +117,9 @@ describe Readability do
         b[:content_score] <=> a[:content_score]
       }.first[:elem][:id].should == "body"
     end
-  end
 
-  describe "score_paragraphs" do
     context "when two consequent br tags are used instead of p" do
-      before :each do
+      it "should assign the higher score to the first paragraph in this particular example" do
         @doc = Readability::Document.new(<<-HTML)
           <html>
             <head>
@@ -140,9 +140,6 @@ describe Readability do
           </html>
         HTML
         @candidates = @doc.score_paragraphs(0)
-      end
-
-      it "should assign the higher score to the first paragraph in this particular example" do
         @candidates.values.sort_by { |a| -a[:content_score] }.first[:elem][:id].should == 'post1'
       end
     end
@@ -204,14 +201,13 @@ describe Readability do
     end
     
     it "should output expected fragments of text" do
-
       checks = 0
       @samples.each do |sample|
         html = File.read(File.dirname(__FILE__) + "/fixtures/samples/#{sample}.html")
         doc = Readability::Document.new(html).content
 
         load "fixtures/samples/#{sample}-fragments.rb"
-        puts "testing #{sample}..."
+        #puts "testing #{sample}..."
         
         $required_fragments.each do |required_text|
           doc.should include(required_text)
@@ -223,7 +219,32 @@ describe Readability do
           checks += 1
         end
       end
-      puts "Performed #{checks} checks."
+      #puts "Performed #{checks} checks."
+    end
+  end
+
+  describe "encoding guessing" do
+    if RUBY_VERSION =~ /^1\.9\./
+      context "with ruby 1.9.2" do
+        it "should correctly guess and enforce HTML encoding" do
+          doc = Readability::Document.new("<html><head><meta http-equiv='content-type' content='text/html; charset=LATIN1'></head><body><div>hi!</div></body></html>")
+          content = doc.content
+          content.encoding.to_s.should == "ISO-8859-1"
+          content.should be_valid_encoding
+        end
+
+        it "should allow encoding guessing to be skipped" do
+          do_not_allow(GuessHtmlEncoding).encode
+          doc = Readability::Document.new(@simple_html_fixture, :do_not_guess_encoding => true)
+          doc.content
+        end
+
+        it "should allow encoding guessing to be overridden" do
+          do_not_allow(GuessHtmlEncoding).encode
+          doc = Readability::Document.new(@simple_html_fixture, :encoding => "UTF-8")
+          doc.content
+        end
+      end
     end
   end
 end

data/spec/spec_helper.rb

@@ -1,10 +1,7 @@
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'rubygems'
 require 'readability'
-require 'spec'
-require 'spec/autorun'
-
-Spec::Runner.configure do |config|
+require 'rr'
 
+RSpec.configure do |config|
+  config.mock_with :rr
 end

metadata

@@ -1,15 +1,10 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: ruby-readability
-version: !ruby/object:Gem::Version 
-  hash: 17
-  prerelease: false
-  segments: 
-  - 0
-  - 2
-  - 3
-  version: 0.2.3
+version: !ruby/object:Gem::Version
+  version: 0.3.0.pre
+  prerelease: 6
 platform: ruby
-authors: 
+authors:
 - Andrew Cantino
 - starrhorne
 - libc
@@ -17,57 +12,68 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-01-26 00:00:00 -08:00
-default_executable: readability
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-10-26 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rspec
+  requirement: &70185897981180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
   prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  version_requirements: *70185897981180
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: &70185897980680 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 13
-        segments: 
-        - 1
-        - 2
-        - 9
-        version: 1.2.9
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.0'
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *70185897980680
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70185897980220 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 1
-        - 4
-        - 2
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.4.2
   type: :runtime
-  version_requirements: *id002
+  prerelease: false
+  version_requirements: *70185897980220
+- !ruby/object:Gem::Dependency
+  name: guess_html_encoding
+  requirement: &70185897979760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *70185897979760
 description: Port of arc90's readability project to ruby
-email: andrew@iterationlabs.com
-executables: 
+email:
+- andrew@iterationlabs.com
+executables:
 - readability
 extensions: []
-
-extra_rdoc_files: 
-- README
-files: 
+extra_rdoc_files: []
+files:
 - .document
+- .gitignore
+- Gemfile
 - README
 - Rakefile
-- VERSION
 - bin/readability
 - lib/readability.rb
+- lib/ruby-readability.rb
 - ruby-readability.gemspec
 - spec/fixtures/cant_read.html
 - spec/fixtures/sample.html
@@ -83,44 +89,42 @@ files:
 - spec/readability_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-has_rdoc: true
 homepage: http://github.com/iterationlabs/ruby-readability
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
 requirements: []
-
-rubyforge_project: 
-rubygems_version: 1.3.7
+rubyforge_project: ruby-readability
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
 summary: Port of arc90's readability project to ruby
-test_files: 
+test_files:
+- spec/fixtures/cant_read.html
+- spec/fixtures/sample.html
 - spec/fixtures/samples/blogpost_with_links-fragments.rb
+- spec/fixtures/samples/blogpost_with_links.html
 - spec/fixtures/samples/channel4-1-fragments.rb
+- spec/fixtures/samples/channel4-1.html
 - spec/fixtures/samples/foxnews-india1-fragments.rb
+- spec/fixtures/samples/foxnews-india1.html
 - spec/fixtures/samples/globemail-ottawa-cuts-fragments.rb
+- spec/fixtures/samples/globemail-ottawa-cuts.html
+- spec/fixtures/should_not_truncate.txt
 - spec/readability_spec.rb
+- spec/spec.opts
 - spec/spec_helper.rb

data/VERSION

@@ -1 +0,0 @@
-0.2.3