pismo 0.7.2 → 0.7.4

data/test/helper.rb

@@ -9,6 +9,9 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'pismo'
 
+# Set time zone to prevent parsed times not matching those stored in metadata_expected.yaml
+ENV['TZ'] = 'UTC'
+
 class Test::Unit::TestCase
   include Pismo
   HTML_DIRECTORY = File.dirname(__FILE__) + "/corpus"

data/test/test_corpus.rb

@@ -11,6 +11,8 @@ class TestCorpus < Test::Unit::TestCase
       # Load the "expected metadata" ready for tests
       @metadata = YAML.load(open(HTML_DIRECTORY + "/metadata_expected.yaml"))
       @reader_metadata = YAML.load(open(HTML_DIRECTORY + "/reader_expected.yaml"))
+      @readers = {}
+      Dir[HTML_DIRECTORY + "/readers/*_expected.yaml"].each { |filename| @readers[File.basename(filename).sub(/_expected\.yaml$/, '').to_sym] = File.read(filename) }
     end
     
     should "pass basic sanitization and result in Nokogiri documents" do
@@ -31,11 +33,22 @@ class TestCorpus < Test::Unit::TestCase
       end
     end
     
-    should "pass content extraction tests" do
+    should "pass base reader content extraction tests" do
       @reader_metadata.each do |file, expected|
-        @doc = Reader::Document.new(@corpus[file])
+        @doc = Reader::Document.create(@corpus[file])
         assert_equal expected, @doc.sentences(2)
       end
-    end    
+    end
+
+    should "pass reader content extraction tests" do
+      @readers.each do |reader, expected|
+        results = YAML.load(expected)
+        results.each_key do |file|
+          @doc = Document.new(@corpus[file], :reader => reader)
+          assert_equal results[file][0..1000], @doc.body[0..1000]
+        end
+      end
+    end
+
   end
 end

metadata

@@ -1,110 +1,109 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: pismo
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 7
-  - 2
-  version: 0.7.2
+version: !ruby/object:Gem::Version
+  version: 0.7.4
 platform: ruby
-authors: 
+authors:
 - Peter Cooper
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2010-07-27 00:00:00 +01:00
-default_executable: pismo
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2010-12-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
-  type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: awesome_print
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: sanitize
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sanitize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: fast-stemmer
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fast-stemmer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: chronic
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: chronic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id006
-description: Pismo extracts and retrieves content-related metadata from HTML pages - you can use the resulting data in an organized way, such as a summary/first paragraph, body text, keywords, RSS feed URL, favicon, etc.
-email: 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Pismo extracts and retrieves content-related metadata from HTML pages
+  - you can use the resulting data in an organized way, such as a summary/first paragraph,
+  body text, keywords, RSS feed URL, favicon, etc.
+email:
 - git@peterc.org
-executables: 
+executables:
 - pismo
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .document
 - .gitignore
 - Gemfile
@@ -118,6 +117,9 @@ files:
 - lib/pismo/external_attributes.rb
 - lib/pismo/internal_attributes.rb
 - lib/pismo/reader.rb
+- lib/pismo/reader/base.rb
+- lib/pismo/reader/cluster.rb
+- lib/pismo/reader/tree.rb
 - lib/pismo/stopwords.txt
 - lib/pismo/version.rb
 - pismo.gemspec
@@ -129,52 +131,45 @@ files:
 - test/corpus/gmane.html
 - test/corpus/huffington.html
 - test/corpus/metadata_expected.yaml
-- test/corpus/metadata_expected.yaml.old
 - test/corpus/queness.html
 - test/corpus/reader_expected.yaml
+- test/corpus/readers/cluster_expected.yaml
+- test/corpus/readers/tree_expected.yaml
 - test/corpus/rubyinside.html
 - test/corpus/rww.html
 - test/corpus/spolsky.html
 - test/corpus/techcrunch.html
+- test/corpus/thegoodbookblog.html
 - test/corpus/tweet.html
 - test/corpus/youtube.html
 - test/corpus/zefrank.html
 - test/helper.rb
 - test/test_corpus.rb
 - test/test_pismo_document.rb
-has_rdoc: true
 homepage: http://github.com/peterc/pismo
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: pismo
-rubygems_version: 1.3.7
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Extracts or retrieves content-related metadata from HTML pages
-test_files: 
+test_files:
 - test/corpus/bbcnews.html
 - test/corpus/bbcnews2.html
 - test/corpus/briancray.html
@@ -183,13 +178,15 @@ test_files:
 - test/corpus/gmane.html
 - test/corpus/huffington.html
 - test/corpus/metadata_expected.yaml
-- test/corpus/metadata_expected.yaml.old
 - test/corpus/queness.html
 - test/corpus/reader_expected.yaml
+- test/corpus/readers/cluster_expected.yaml
+- test/corpus/readers/tree_expected.yaml
 - test/corpus/rubyinside.html
 - test/corpus/rww.html
 - test/corpus/spolsky.html
 - test/corpus/techcrunch.html
+- test/corpus/thegoodbookblog.html
 - test/corpus/tweet.html
 - test/corpus/youtube.html
 - test/corpus/zefrank.html

data/test/corpus/metadata_expected.yaml.old

@@ -1,122 +0,0 @@
---- 
-:rww: 
-  :title: "Cartoon: Apple Tablet: Now With Barometer and Bird Call Generator"
-  :feed: http://www.readwriteweb.com/rss.xml
-:briancray: 
-  :title: 5 great examples of popular blog posts that you should know
-  :feed: http://feeds.feedburner.com/briancray/blog
-  :lede: "This is a mock post. While there is a place for all of these posts, I'm trying to make a point that original blogs are being shut out by formulaic blogs."
-:huffington:
-  :title: Afghans Losing Hope After 8 Years Of War
-  :author: TODD PITMAN
-  :feed: http://feeds.huffingtonpost.com/huffingtonpost/raw_feed
-  :lede: "KABUL - The man on the motorcycle was going the wrong way down a one-way street, gesturing indignantly for the phalanx of traffic-clogged cars in front of him to move."
-:bbcnews:
-  :title: Gay Muslims made homeless by family violence
-  :author: Poonam Taneja
-  :description: A charity is dealing with more gay Muslims made homeless after fleeing forced marriages and so-called "honour" violence.
-  :lede: A UK charity is dealing with an increasing number of young gay Muslims becoming homeless after fleeing forced marriages and so-called honour violence.
-  :feed: http://newsrss.bbc.co.uk/rss/newsonline_uk_edition/england/rss.xml
-:factor:
-  :title: Factor's bootstrap process explained
-  :lede: "Separation of concerns between Factor VM and library codeThe Factor VM implements an abstract machine consisting of a data heap of objects, a code heap of machine code blocks, and a set of stacks. The VM loads an image file on startup, which becomes the data and code heap. "
-:youtube:
-  :title: YMO - Rydeen (Official Video)
-  :author: ymo1965
-:spolsky:
-  :title: The Absolute Minimum Every Software Developer Absolutely, Positively Must Know About Unicode and Character Sets (No Excuses!)
-  :description: Haven't mastered the basics of Unicode and character sets? Please don't write another line of code until you've read this article.
-  :author: Joel Spolsky
-  :favicon: /favicon.ico
-  :feed: http://www.joelonsoftware.com/rss.xml
-:techcrunch: 
-  :title: Googlle Gets A Sexy New Logo; Remains Sketchy
-  :author: MG Siegler
-  :keywords:
-  - - googlle
-    - 35
-  - - logo
-    - 10
-  - - google
-    - 6
-  - - site
-    - 3
-  - - font
-    - 2
-  - - india
-    - 2
-  - - surprised
-    - 1
-  - - week
-    - 1
-  - - switched
-    - 1
-  - - school
-    - 1
-  - - things
-    - 1
-  - - removing
-    - 1
-  - - steve
-    - 1
-  - - decided
-    - 1
-  - - advantage
-    - 1
-  - - wasn
-    - 1
-  - - accepting
-    - 1
-  - - red
-    - 1
-  - - copy
-    - 1
-  - - wouldn
-    - 1
-:rubyinside: 
-  :title: "CoffeeScript: A New Language With A Pure Ruby Compiler"
-  :author: Peter Cooper
-  :lede: CoffeeScript (GitHub repo) is a new programming language with a pure Ruby compiler. Creator Jeremy Ashkenas calls it "JavaScript's less ostentatious kid brother" - mostly because it compiles into JavaScript and shares most of the same constructs, but with a different, tighter syntax.
-  :body: "CoffeeScript (GitHub repo) is a new programming language with a pure Ruby compiler. Creator Jeremy Ashkenas calls it \"JavaScript's less ostentatious kid brother\" - mostly because it compiles into JavaScript and shares most of the same constructs, but with a different, tighter syntax.\nTo get a feel for the language, check out this example code (CoffeeScript on the left, resulting JavaScript on the right):\nAs a Ruby project, you can get the CoffeeScript compiler installed with a simple gem install coffee-script or check out the code from/on GitHub. The code is worth a look as it's notably quite vanilla with hand crafted Ruby covering the lexer and code generation and Racc built code for the parser.\n"
-  :feed: http://www.rubyinside.com/feed/
-  :keywords:
-  - - ruby
-    - 15
-  - - coffeescript
-    - 15
-  - - compiler
-    - 10
-  - - language
-    - 10
-  - - coffee
-    - 5
-  - - pure
-    - 5
-  - - code
-    - 5
-  - - script
-    - 5
-  - - javascript
-    - 3
-  - - github
-    - 2
-  - - syntax
-    - 1
-  - - programming
-    - 1
-  - - brother
-    - 1
-  - - constructs
-    - 1
-  - - vanilla
-    - 1
-  - - parser
-    - 1
-  - - lexer
-    - 1
-  - - project
-    - 1
-  - - installed
-    - 1
-  - - simple
-    - 1