pismo 0.6.0 → 0.6.1

data/README.markdown

@@ -26,7 +26,9 @@ There's also a shorter "convenience" method which might be handy in IRB - it doe
 
     Pismo['http://www.rubyflow.com/items/4082'].title   # => "Install Ruby as a non-root User"
     
-The current metadata methods are #title, #titles, #author, #authors, #lede, #keywords, #sentences(qty), #body, #feed, #feeds, #favicon, #description and #datetime. These are not fully documented here yet, you'll just need to try them out. The plural methods like #titles, #authors, and #feeds will return multiple matches in an array, if present. This is so you can use your own techniques to choose a "best" result in ambiguous cases.
+The current metadata methods are #title, #titles, #author, #authors, #lede, #keywords, #sentences(qty), #body, #html_body, #feed, #feeds, #favicon, #description and #datetime. These are not fully documented here yet, you'll just need to try them out. The plural methods like #titles, #authors, and #feeds will return multiple matches in an array, if present. This is so you can use your own techniques to choose a "best" result in ambiguous cases.
+
+The html_body and body methods will be of particular interest. They return the "body" of the page as determined by Pismo's "Reader" (like Arc90's Readability or Safari Reader) algorithm. #body returns it as plain-text, #html_body maintains some basic HTML styling.
     
 ## CAUTIONS / WARNINGS:
 

data/VERSION

@@ -1 +1 @@
-0.6.0
+0.6.1

data/lib/pismo/internal_attributes.rb

@@ -189,12 +189,12 @@ module Pismo
                   '.post-text p',
                   '#blogpost p',
                   '.story-teaser',
-                  '.subhead',
                   '//div[@class="entrytext"]//p[string-length()>10]',                      # Ruby Inside / Kubrick style
                   'section p',
                   '.entry .text p',
                   '.entry-content p',
                   '#wikicontent p',                                                        # Google Code style
+                  '.wikistyle p',                                                          # GitHub style
                   '//td[@class="storybody"]/p[string-length()>10]',                        # BBC News style
                   '//div[@class="entry"]//p[string-length()>100]',
                   # The below is a horrible, horrible way to pluck out lead paras from crappy Blogspot blogs that
@@ -206,16 +206,15 @@ module Pismo
                   '#article p',
                   '.post-body',
                   '.entry-content',
-                  '.body p',
                   '.document_description_short p',    # Scribd
                   '.single-post p'
                   ], all)
 
       # TODO: Improve sentence extraction - this is dire even if it "works for now"
       if lede && String === lede
-        return lede[/^(.*?[\.\!\?]\s){2}/m] || lede
+        return (lede[/^(.*?[\.\!\?]\s){2}/m] || lede).to_s.strip
       elsif lede && Array === lede
-        return lede.map { |l| l.to_s[/^(.*?[\.\!\?]\s){2}/m] || l }.uniq
+        return lede.map { |l| l.to_s[/^(.*?[\.\!\?]\s){2}/m].strip || l }.uniq
       else
         return reader_doc && !reader_doc.sentences(2).empty? ? reader_doc.sentences(2).join(' ') : nil
       end
@@ -268,7 +267,12 @@ module Pismo
     
     # Returns body text as determined by Reader algorithm
     def body
-      @body ||= reader_doc.content.strip      
+      @body ||= reader_doc.content(true).strip      
+    end
+    
+    # Returns body text as determined by Reader algorithm WITH basic HTML formatting intact
+    def html_body
+      @html_body ||= reader_doc.content.strip      
     end
     
     # Returns URL to the site's favicon

data/pismo.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{pismo}
-  s.version = "0.6.0"
+  s.version = "0.6.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Peter Cooper"]
@@ -59,7 +59,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/peterc/pismo}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.3.5}
   s.summary = %q{Extracts or retrieves content-related metadata from HTML pages}
   s.test_files = [
     "test/helper.rb",
@@ -71,7 +71,7 @@ Gem::Specification.new do |s|
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<awesome_print>, [">= 0"])
       s.add_runtime_dependency(%q<jeweler>, [">= 0"])

data/test/corpus/metadata_expected.yaml

@@ -31,9 +31,9 @@
   :feed: http://newsrss.bbc.co.uk/rss/newsonline_uk_edition/england/rss.xml
 :factor:
   :title: Factor's bootstrap process explained
-  :lede: "Separation of concerns between Factor VM and library codeThe Factor VM implements an abstract machine consisting of a data heap of objects, a code heap of machine code blocks, and a set of stacks. The VM loads an image file on startup, which becomes the data and code heap. "
+  :lede: "Separation of concerns between Factor VM and library codeThe Factor VM implements an abstract machine consisting of a data heap of objects, a code heap of machine code blocks, and a set of stacks. The VM loads an image file on startup, which becomes the data and code heap."
   :ledes:
-    - "Separation of concerns between Factor VM and library codeThe Factor VM implements an abstract machine consisting of a data heap of objects, a code heap of machine code blocks, and a set of stacks. The VM loads an image file on startup, which becomes the data and code heap. "
+    - "Separation of concerns between Factor VM and library codeThe Factor VM implements an abstract machine consisting of a data heap of objects, a code heap of machine code blocks, and a set of stacks. The VM loads an image file on startup, which becomes the data and code heap."
 :youtube:
   :title: YMO - Rydeen (Official Video)
   :author: ymo1965
@@ -68,6 +68,6 @@
   :sentences: I am pleased to report that the GCC Steering Committee and the FSF have approved the use of C++ in GCC itself. Of course, there's no reason for us to use C++ features just because we can. The goal is a better compiler for users, not a C++ code base for its own sake.
 :queness:
   :title: 18 Incredible CSS3 Effects You Have Never Seen Before
-  :lede: "CSS3 is hot these days and will soon be available in most modern browser. Just recently, I started to become aware to the present of CSS3 around the web. "
+  :lede: "CSS3 is hot these days and will soon be available in most modern browser. Just recently, I started to become aware to the present of CSS3 around the web."
   :sentences: CSS3 is hot these days and will soon be available in most modern browser. Just recently, I started to become aware to the present of CSS3 around the web. I can see some of the websites such as twitter and designer portfolios websites are using it.
   :datetime: 2010-06-02 12:00:00 +01:00

metadata

@@ -1,13 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: pismo
 version: !ruby/object:Gem::Version 
-  hash: 7
-  prerelease: false
-  segments: 
-  - 0
-  - 6
-  - 0
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors: 
 - Peter Cooper
@@ -20,102 +14,74 @@ default_executable: pismo
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: shoulda
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
-  type: :development
-  version_requirements: *id001
+    version: 
 - !ruby/object:Gem::Dependency 
   name: awesome_print
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
-  type: :development
-  version_requirements: *id002
+    version: 
 - !ruby/object:Gem::Dependency 
   name: jeweler
-  prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
-  type: :runtime
-  version_requirements: *id003
+    version: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
-  prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
-  type: :runtime
-  version_requirements: *id004
+    version: 
 - !ruby/object:Gem::Dependency 
   name: sanitize
-  prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
-  type: :runtime
-  version_requirements: *id005
+    version: 
 - !ruby/object:Gem::Dependency 
   name: fast-stemmer
-  prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
-  type: :runtime
-  version_requirements: *id006
+    version: 
 - !ruby/object:Gem::Dependency 
   name: chronic
-  prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
-  type: :runtime
-  version_requirements: *id007
+    version: 
 description: Pismo extracts and retrieves content-related metadata from HTML pages - you can use the resulting data in an organized way, such as a summary/first paragraph, body text, keywords, RSS feed URL, favicon, etc.
 email: git@peterc.org
 executables: 
@@ -172,27 +138,21 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
+  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
+  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: Extracts or retrieves content-related metadata from HTML pages