RubyGems - pismo - Versions diffs - 0.2.0 - Mend

pismo 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (30) hide show

data/.document +5 -0
data/.gitignore +21 -0
data/LICENSE +32 -0
data/README.rdoc +68 -0
data/Rakefile +95 -0
data/VERSION +1 -0
data/bin/pismo +36 -0
data/lib/pismo/document.rb +50 -0
data/lib/pismo/external_attributes.rb +14 -0
data/lib/pismo/internal_attributes.rb +202 -0
data/lib/pismo/readability.rb +316 -0
data/lib/pismo/stopwords.txt +893 -0
data/lib/pismo.rb +44 -0
data/pismo.gemspec +92 -0
data/test/corpus/bbcnews.html +2131 -0
data/test/corpus/briancray.html +269 -0
data/test/corpus/cant_read.html +426 -0
data/test/corpus/factor.html +1362 -0
data/test/corpus/huffington.html +2932 -0
data/test/corpus/metadata_expected.yaml +81 -0
data/test/corpus/rubyinside.html +318 -0
data/test/corpus/rww.html +1351 -0
data/test/corpus/spolsky.html +298 -0
data/test/corpus/techcrunch.html +1285 -0
data/test/corpus/youtube.html +2348 -0
data/test/helper.rb +15 -0
data/test/test_corpus.rb +33 -0
data/test/test_pismo_document.rb +34 -0
data/test/test_readability.rb +152 -0
metadata +146 -0

data/test/helper.rb ADDED Viewed

@@ -0,0 +1,15 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'open-uri'
+require 'yaml'
+begin; require 'turn'; rescue LoadError; end
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'pismo'
+class Test::Unit::TestCase
+  include Pismo
+  HTML_DIRECTORY = File.dirname(__FILE__) + "/corpus"
+end

data/test/test_corpus.rb ADDED Viewed

@@ -0,0 +1,33 @@
+require 'helper'
+class TestCorpus < Test::Unit::TestCase
+  context "A corpus of HTML documents" do
+    setup do
+      # Load the corpus files' HTML content into a hash
+      @corpus = {}
+      Dir[HTML_DIRECTORY + "/*.html"].each { |filename| @corpus[File.basename(filename).sub(/\.html$/, '').to_sym] = File.read(filename) }
+      # Load the "expected metadata" ready for tests
+      @metadata = YAML.load(open(HTML_DIRECTORY + "/metadata_expected.yaml"))
+    end
+    should "pass basic sanitization and result in Nokogiri documents" do
+      @corpus.values.each do |html|
+        doc = Document.new(html)
+        assert doc.html.length > 1000
+        assert doc.doc.kind_of?(Nokogiri::HTML::Document)
+      end
+    end
+    should "pass metadata extraction tests" do
+      @metadata.each do |file, expected|
+        @doc = Document.new(@corpus[file])
+        expected.each do |k, v|
+          assert_equal v, @doc.send(k)
+        end
+      end
+    end
+  end
+end

data/test/test_pismo_document.rb ADDED Viewed

@@ -0,0 +1,34 @@
+require 'helper'
+class TestPismoDocument < Test::Unit::TestCase
+  context "Pismo::Document" do
+    should "process an IO/File object" do
+      doc = Document.new(open(HTML_DIRECTORY + "/rubyinside.html"))
+      assert doc.doc.kind_of?(Nokogiri::HTML::Document)
+    end
+  end
+  context "A very basic Pismo document" do
+    setup do
+      @doc = Document.new(%{<html><body><h1>Hello</h1></body></html>})
+    end
+    should "pass sanitization" do
+      assert_equal %{<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">\n<html><body><h1>Hello</h1></body></html>\n}, @doc.html
+    end
+    should "result in a Nokogiri document" do
+      assert @doc.doc.kind_of?(Nokogiri::HTML::Document)
+    end
+  end
+  context "A basic real world blog post" do
+    setup do
+      @doc = Document.new(open(HTML_DIRECTORY + "/rubyinside.html"))
+    end
+    should "provide a title" do
+      assert_equal  "CoffeeScript: A New Language With A Pure Ruby Compiler", @doc.title
+    end
+  end
+end

data/test/test_readability.rb ADDED Viewed

@@ -0,0 +1,152 @@
+require 'helper'
+class TestReadability < Test::Unit::TestCase
+  context "Readability" do
+    setup do
+      @simple_html_fixture = <<-HTML
+        <html>
+          <head>
+            <title>title!</title>
+          </head>
+          <body class='comment'>
+            <div>
+              <p class='comment'>a comment</p>
+              <div class='comment' id='body'>real content</div>
+              <div id="contains_blockquote"><blockquote>something in a table</blockquote></div>
+            </div>
+          </body>
+        </html>
+      HTML
+    end
+    context "transformMisusedDivsIntoParagraphs" do
+      setup do
+        @doc = Readability::Document.new(@simple_html_fixture)
+        @doc.transform_misused_divs_into_paragraphs!
+      end
+      should "transform divs containing no block elements into <p>s" do
+        assert_equal "p", @doc.html.css("#body").first.name
+      end
+      should "not transform divs that contain block elements" do
+        assert_equal "div", @doc.html.css("#contains_blockquote").first.name
+      end
+    end
+    context "score_node" do
+      setup do
+        @doc = Readability::Document.new(<<-HTML)
+          <html>
+            <body>
+              <div id='elem1'>
+                <p>some content</p>
+              </div>
+              <th id='elem2'>
+                <p>some other content</p>
+              </th>
+            </body>
+          </html>
+        HTML
+        @elem1 = @doc.html.css("#elem1").first
+        @elem2 = @doc.html.css("#elem2").first
+      end
+      should "like <div>s more than <th>s" do
+        assert @doc.score_node(@elem1)[:content_score] > @doc.score_node(@elem2)[:content_score]
+      end
+      should "like classes like text more than classes like comment" do
+        @elem2.name = "div"
+        assert_equal @doc.score_node(@elem2)[:content_score], @doc.score_node(@elem1)[:content_score]
+        @elem1['class'] = "text"
+        @elem2['class'] = "comment"
+        assert @doc.score_node(@elem1)[:content_score] > @doc.score_node(@elem2)[:content_score]
+      end
+    end
+    context "remove_unlikely_candidates!" do
+      setup do
+        @doc = Readability::Document.new(@simple_html_fixture)
+        @doc.remove_unlikely_candidates!
+      end
+      should "remove things that have class comment" do
+        assert @doc.html.inner_html !~ /a comment/
+      end
+      should "not remove body tags" do
+        assert @doc.html.inner_html =~ /<\/body>/
+      end
+      should "not remove things with class comment and id body" do
+        assert @doc.html.inner_html =~ /real content/
+      end
+    end
+    context "score_paragraphs" do
+      setup do
+        @doc = Readability::Document.new(%{
+          <html>
+            <head>
+              <title>title!</title>
+            </head>
+            <body id="body">
+              <div id="div1">
+                <div id="div2>
+                  <p id="some_comment">a comment</p>
+                </div>
+                <p id="some_text">some text</p>
+              </div>
+              <div id="div3">
+                <p id="some_text2">some more text</p>
+              </div>
+            </body>
+          </html>
+        })
+        @candidates = @doc.score_paragraphs(0)
+      end
+      should "score elements in the document" do
+        assert_equal 3, @candidates.values.length
+      end
+      should "prefer the body in this particular example" do
+        assert_equal "body", @candidates.values.sort { |a, b|
+          b[:content_score] <=> a[:content_score]
+        }.first[:elem][:id]
+      end
+    end
+    context "the cant_read.html fixture" do
+      should "work on the cant_read.html fixture with some allowed tags" do
+        allowed_tags = %w[div span table tr td p i strong u h1 h2 h3 h4 pre code br a]
+        allowed_attributes = %w[href]
+        html = File.read(HTML_DIRECTORY + "/cant_read.html")
+        assert Readability::Document.new(html, :tags => allowed_tags, :attributes => allowed_attributes).content.match(/Can you talk a little about how you developed the looks for the/)
+      end
+    end
+    context "general functionality" do
+      setup do
+        @doc = Readability::Document.new("<html><head><title>title!</title></head><body><div><p>Some content</p></div></body>",
+                                         :min_text_length => 0, :retry_length => 1)
+      end
+      should "return the main page content" do
+        assert @doc.content.match("Some content")
+      end
+    end
+    context "ignoring sidebars" do
+      setup do
+        @doc = Readability::Document.new("<html><head><title>title!</title></head><body><div><p>Some content</p></div><div class='sidebar'><p>sidebar<p></div></body>",
+                                         :min_text_length => 0, :retry_length => 1)
+      end
+      should "not return the sidebar" do
+        assert !@doc.content.match("sidebar")
+      end
+    end
+  end
+end

metadata ADDED Viewed

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: pismo
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Peter Cooper
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2010-03-26 00:00:00 +00:00
+default_executable: pismo
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  type: :development
+  version_requirement:
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: "0"
+    version:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  type: :runtime
+  version_requirement:
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: "0"
+    version:
+- !ruby/object:Gem::Dependency
+  name: loofah
+  type: :runtime
+  version_requirement:
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: "0"
+    version:
+- !ruby/object:Gem::Dependency
+  name: httparty
+  type: :runtime
+  version_requirement:
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: "0"
+    version:
+- !ruby/object:Gem::Dependency
+  name: fast-stemmer
+  type: :runtime
+  version_requirement:
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: "0"
+    version:
+- !ruby/object:Gem::Dependency
+  name: chronic
+  type: :runtime
+  version_requirement:
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: "0"
+    version:
+description: Pismo extracts and retrieves content-related metadata from HTML pages - you can use the resulting data in an organized way, such as a summary/first paragraph, del.icio.us tags, first image used in the content block, etc.
+email: git@peterc.org
+executables:
+- pismo
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.rdoc
+files:
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- bin/pismo
+- lib/pismo.rb
+- lib/pismo/document.rb
+- lib/pismo/external_attributes.rb
+- lib/pismo/internal_attributes.rb
+- lib/pismo/readability.rb
+- lib/pismo/stopwords.txt
+- pismo.gemspec
+- test/corpus/bbcnews.html
+- test/corpus/briancray.html
+- test/corpus/cant_read.html
+- test/corpus/factor.html
+- test/corpus/huffington.html
+- test/corpus/metadata_expected.yaml
+- test/corpus/rubyinside.html
+- test/corpus/rww.html
+- test/corpus/spolsky.html
+- test/corpus/techcrunch.html
+- test/corpus/youtube.html
+- test/helper.rb
+- test/test_corpus.rb
+- test/test_pismo_document.rb
+- test/test_readability.rb
+has_rdoc: true
+homepage: http://github.com/peterc/pismo
+licenses: []
+post_install_message:
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+requirements: []
+rubyforge_project:
+rubygems_version: 1.3.5
+signing_key:
+specification_version: 3
+summary: Extracts or retrieves content-related metadata from HTML pages
+test_files:
+- test/helper.rb
+- test/test_corpus.rb
+- test/test_pismo_document.rb
+- test/test_readability.rb