rfeedparser 0.9.940 → 0.9.950

data/README

@@ -1,27 +1,50 @@
-Universal Feed Parser
-Parse RSS and Atom feeds in Python.  3000 unit tests.  Open source.
-
-Copyright (c) 2002-5 by Mark Pilgrim
-open source, see LICENSE file for details
-
------
-
-To use:
-  If installed as a gem
-     require 'rubygems'
-     require 'rfeedparser'
-
-     fp = FeedParser.parse("some-feed-filepath-or-url")
-
-  If not installed as a gem, copy the contents of lib into the ruby path and just use 
-     require 'rfeedparser'
-     
-     fp = FeedParser.parse("some-feed-filepath-or-url")
-
-----
-
-For developers: 
-  I currently have the "rough" code in a bzr branch over at
-  <http://somethingsimilar.com/code/bzr/rfeedparser/>.  You'll want
-  to check out the rfeedparser-main branch for the current code, or
-  rfeedparser-release for the code in the latest release.
+= Universal Feed Parser
+  http://rfeedparser.rubyforge.org
+  Original Python code Copyright (c) 2002-5 by Mark Pilgrim
+  Ruby port by Jeff Hodges.
+
+== Description
+
+Parse nearly any RSS and Atom feeds in Ruby.  3000 unit tests.  Open source.
+
+== Installation
+
+For Debian-based systems:
+
+  $ sudo apt-get install libxml-ruby1.8 # or libxml-parser-ruby1.8
+
+TODO: dependency installation instructions for other platforms.
+
+And then:
+
+  $ sudo gem install rfeedparser
+
+Or for the latest development version:
+
+  $ git clone git://github.com/technomancy/rfeedparser.git
+
+Dependencies on other gems are handled by rubygems, but rfeedparser
+also relies on having bindings to a native XML parsing library. The
+recommended choice is libxml, which is installable with the
+libxml-ruby1.8 package in Debian-based systems. But it can also fall
+back to expat (the libxml-parser-ruby1.8 Debian package) if libxml is
+not installed.
+
+== Usage
+
+   require 'rubygems' # may omit if installed manually
+   require 'rfeedparser'
+
+   feed = FeedParser.parse("some-feed-stream-filepath-or-url")
+
+   feed.entries.each do |e|
+     puts e.title
+     puts e.content
+     puts e.published
+     puts '----'
+   end
+
+== Contributing
+
+Clone the git repository at git://github.com/jmhodges/rfeedparser.git and
+which has a webpage at http://github.com/jmhodges/rfeedparser/tree/master.

data/RUBY-TESTING

@@ -1,15 +1,18 @@
 === Testing rFeedParser ===
-    Simply run `ruby tests/rfeedparsertest.rb` to run all of the FeedParser tests.
+Simply run rake to run all of the FeedParser tests.
+
 Optionally, you can start up rfeedparserserver.rb and, in another shell, run rfeedparser.rb
 against "http://localhost:8097/tests/path/to/testcase.xml" if you want
 to try a test individually.  You can, of course, run rfeedparser.rb with
 the file path to the testcase, but there may be HTTP headers required for
 a successful run that can only be given by running rfeedparser against
 the HTTP server path. I'll probably merge feedparserserver.rb into
-rfeedparsertest.rb soon.
+rfeedparsertest.rb soon.  Note that there are many test failures that occur
+solely because of problems with a platforms iconv installation.  Inconsistency
+in iconv is the leading cause of errors in rfp's test cases.
 
 === Last Count 20070321 ===
-    By my last count, rfeedparsertest.rb says that there are 45 assertions
+By my last count, rfeedparsertest.rb says that there are 45 assertions
 that fail, and 4 that error out. I've included here a few tests that
 "Failed, Sort Of".  By that I mean, the behaviors the tests are meant to
 check are correct, but the test fails because of some other superficial

data/lib/rfeedparser.rb

@@ -1,103 +1,107 @@
 #!/usr/bin/env ruby
-"""Universal feed parser in Ruby
+# Universal feed parser in Ruby
+#
+# Handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds
+#
+# Visit http://feedparser.org/ for the latest version in Python
+# Visit http://feedparser.org/docs/ for the latest documentation
+# Email Jeff Hodges at jeff@somethingsimilar.com with questions
+#
+# Required: Ruby 1.8
 
-Handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds
-
-Visit http://feedparser.org/ for the latest version in Python
-Visit http://feedparser.org/docs/ for the latest documentation
-Email Jeff Hodges at jeff@obquo.com for questions
-
-Required: Ruby 1.8
-"""
 $KCODE = 'UTF8'
 require 'stringio'
 require 'uri'
+require 'open-uri'
 require 'cgi' # escaping html
 require 'time'
 require 'pp'
-require 'rubygems'
 require 'base64'
 require 'iconv'
+require 'zlib'
+
+require 'rubygems'
 
+
+# If available, Nikolai's UTF-8 library will ease use of utf-8 documents.
+# See http://git.bitwi.se/ruby-character-encodings.git/.
+begin
+  gem 'character-encodings', ">=0.2.0"
+  require 'encoding/character/utf-8'
+rescue LoadError
+end
+
+# TODO: require these in the files that need them, not in the toplevel
 gem 'hpricot', "=0.6"
 require 'hpricot'
-gem 'character-encodings', ">=0.2.0"
+
 gem 'htmltools', ">=1.10"
+require 'html/sgml-parser'
+
 gem 'htmlentities', ">=4.0.0"
-gem 'activesupport', ">=1.4.1"
-gem 'rchardet', ">=1.0"
+require 'htmlentities'
 
-require 'xml/saxdriver' # calling expat through the xmlparser gem
+gem 'addressable', ">= 1.0.4"
+require 'addressable/uri'
 
+gem 'rchardet', ">=1.0"
 require 'rchardet'
 $chardet = true
 
-require 'encoding/character/utf-8'
-require 'html/sgml-parser'
-require 'htmlentities'
-require 'active_support'
-require 'open-uri'
-include OpenURI
-
 $debug = false
 $compatible = true
 
-$LOAD_PATH << File.expand_path(File.dirname(__FILE__))
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
 require 'rfeedparser/utilities'
-require 'rfeedparser/forgiving_uri'
 require 'rfeedparser/better_sgmlparser'
 require 'rfeedparser/better_attributelist'
 require 'rfeedparser/feedparserdict'
 require 'rfeedparser/parser_mixin'
-require 'rfeedparser/parsers'
 
+require 'rfeedparser/loose_feed_parser'
 
+begin
+  require 'rfeedparser/expat_parser'
+  StrictFeedParser = FeedParser::Expat::StrictFeedParser
+  
+rescue LoadError, NameError
+  STDERR.puts "Could not load expat; trying libxml."
+  
+  begin
+    require 'rfeedparser/libxml_parser'
+    StrictFeedParser = FeedParser::LibXML::StrictFeedParser
 
+  rescue LoadError, NameError
+    StrictFeedParser = nil
+    STDERR.puts "Could not load libxml either; will use loose parser."
+  end
+end
+
+require 'rfeedparser/monkey_patches'
 
 module FeedParser
   extend FeedParserUtilities
   
-  Version = "0.9.940"
-
-  License = """Copyright (c) 2002-2006, Mark Pilgrim, All rights reserved.
-
-  Redistribution and use in source and binary forms, with or without modification,
-  are permitted provided that the following conditions are met:
-
-  * Redistributions of source code must retain the above copyright notice,
-  this list of conditions and the following disclaimer.
-  * Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
-  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-  POSSIBILITY OF SUCH DAMAGE."""
-
-  Translator_From_Python_To_Ruby = "Jeff Hodges <http://somethingsimilar.com>"
-  Author = "Mark Pilgrim <http://diveintomark.org/>"
-  Contributors = [  "Jason Diamond <http://injektilo.org/>",
-    "John Beimler <http://john.beimler.org/>",
-    "Fazal Majid <http://www.majid.info/mylos/weblog/>",
-    "Aaron Swartz <http://aaronsw.com/>",
-    "Kevin Marks <http://epeus.blogspot.com/>",
-    "Jesse Newland <http://jnewland.com/>"
-  ]
+  VERSION = "0.9.950"
+
+  AUTHOR = "Mark Pilgrim <http://diveintomark.org/>"
+  PORTER = "Jeff Hodges <http://somethingsimilar.com>"
+  CONTRIBUTERS = ["Jason Diamond <http://injektilo.org/>",
+                  "John Beimler <http://john.beimler.org/>",
+                  "Fazal Majid <http://www.majid.info/mylos/weblog/>",
+                  "Aaron Swartz <http://aaronsw.com/>",
+                  "Kevin Marks <http://epeus.blogspot.com/>",
+                  "Jesse Newland <http://jnewland.com/>",
+                  "Charlie Savage <http://cfis.savagexi.com/>",
+                  "Phil Hagelberg <http://technomancy.us>"]
+
   # HTTP "User-Agent" header to send to servers when downloading feeds.
   # If you are embedding feedparser in a larger application, you should
   # change this to your application name and URL.
-  USER_AGENT = "rFeedParser/#{Version} +http://rfeedparser.rubyforge.org/"
+  USER_AGENT = "rFeedParser/#{VERSION} +http://rfeedparser.rubyforge.org/"
 
   # HTTP "Accept" header to send to servers when downloading feeds.  If you don't
-  # want to send an Accept header, set this to None.
+  # want to send an Accept header, set this to nil.
   ACCEPT_HEADER = "application/atom+xml,application/rdf+xml,application/rss+xml,application/x-netcdf,application/xml;q=0.9,text/xml;q=0.2,*/*;q=0.1"
 
 
@@ -112,7 +116,7 @@ module FeedParser
 
 
   # ---------- don't touch these ----------
-  class ThingsNobodyCaresAboutButMe < Exception
+  class ThingsNobodyCaresAboutButMe < StandardError
   end
   class CharacterEncodingOverride < ThingsNobodyCaresAboutButMe
   end
@@ -120,7 +124,7 @@ module FeedParser
   end
   class NonXMLContentType < ThingsNobodyCaresAboutButMe
   end
-  class UndeclaredNamespace < Exception
+  class UndeclaredNamespace < StandardError
   end
 
 
@@ -158,14 +162,14 @@ module FeedParser
     url_file_stream_or_string.strip!
     
     
-    furi = ForgivingURI.parse(url_file_stream_or_string)
-    if furi && ['http','https','ftp'].include?(furi.scheme)
+    uri = Addressable::URI.parse(url_file_stream_or_string)
+    if uri && ['http','https','ftp'].include?(uri.scheme)
       auth = nil
 
-      if furi.host && furi.password
-        auth = Base64::encode64("#{furi.user}:#{furi.password}").strip
-        furi.password = nil
-        url_file_stream_or_string = furi.to_s
+      if uri.host && uri.password
+        auth = Base64::encode64("#{uri.user}:#{uri.password}").strip
+        uri.password = nil
+        url_file_stream_or_string = uri.to_s
       end
 
       req_headers = {} 
@@ -212,7 +216,7 @@ module FeedParser
     
     # Use the default compatibility if compatible is nil
     $compatible = options[:compatible].nil? ? $compatible : options[:compatible]
-    
+
     strictklass = options[:strict] || StrictFeedParser
     looseklass = options[:loose] || LooseFeedParser
     options[:handlers] = options[:handlers] || []
@@ -233,7 +237,7 @@ module FeedParser
       f = nil
     end
     
-    if f and !data.blank? and f.respond_to?(:meta)
+    if f and !(data.nil? || data.empty?) and f.respond_to?(:meta)
       # if feed is gzip-compressed, decompress it
       if f.meta['content-encoding'] == 'gzip'
         begin
@@ -285,17 +289,20 @@ module FeedParser
     # - result['encoding'] is the actual encoding, as per RFC 3023 and a variety of other conflicting specifications
     http_headers = result['headers'] || {}
     result['encoding'], http_encoding, xml_encoding, sniffed_xml_encoding, acceptable_content_type =
-    getCharacterEncoding(http_headers,data)
+    getCharacterEncoding(http_headers, data)
+
 
-    if not http_headers.blank? and not acceptable_content_type
-      unless http_headers['content-type'].nil?
+    if !(http_headers.nil? || http_headers.empty?) && !acceptable_content_type
+      if http_headers['content-type']
         bozo_message = "#{http_headers['content-type']} is not an XML media type"
       else
         bozo_message = 'no Content-type specified'
       end
+
       result['bozo'] = true
       result['bozo_exception'] = NonXMLContentType.new(bozo_message) # I get to care about this, cuz Mark says I should.
     end
+
     result['version'], data = stripDoctype(data)
     
     baseuri = http_headers['content-location'] || result['href']
@@ -331,6 +338,7 @@ module FeedParser
       rescue
       end
     end
+
     # if no luck and we have auto-detection library, try that
     if not known_encoding and $chardet
       begin 
@@ -344,8 +352,6 @@ module FeedParser
       end
     end
 
-
-
     # if still no luck and we haven't tried utf-8 yet, try that
     if not known_encoding and not tried_encodings.include?'utf-8'
       begin
@@ -356,6 +362,7 @@ module FeedParser
       rescue
       end
     end
+
     # if still no luck and we haven't tried windows-1252 yet, try that
     if not known_encoding and not tried_encodings.include?'windows-1252'
       begin
@@ -366,7 +373,7 @@ module FeedParser
       rescue
       end
     end
-    
+
     # NOTE this isn't in FeedParser.py 4.1
     # if still no luck and we haven't tried iso-8859-2 yet, try that.
     #if not known_encoding and not tried_encodings.include?'iso-8859-2'
@@ -391,34 +398,28 @@ module FeedParser
       result['encoding'] = proposed_encoding
     end
 
+    use_strict_parser = false unless StrictFeedParser
+
     if use_strict_parser
-      # initialize the SAX parser
-      saxparser = XML::SAX::Helpers::ParserFactory.makeParser("XML::Parser::SAXDriver")
-      feedparser = strictklass.new(baseuri, baselang, 'utf-8')
-      saxparser.setDocumentHandler(feedparser)
-      saxparser.setDTDHandler(feedparser)
-      saxparser.setEntityResolver(feedparser)
-      saxparser.setErrorHandler(feedparser)
-
-      inputdata = XML::SAX::InputSource.new('parsedfeed')
-      inputdata.setByteStream(StringIO.new(data))
       begin
-        saxparser.parse(inputdata)
-      rescue Exception => parseerr # resparse
-        if $debug
-          $stderr << "xml parsing failed\n"
-          $stderr << parseerr.to_s+"\n" # Hrmph.
-        end
+        parser = StrictFeedParser.new(baseuri, baselang)
+        feedparser = parser.handler
+        parser.parse(data)
+
+      rescue => err
+        $stderr << "xml parsing failed: #{err.message}\n#{err.backtrace.join("\n")}" if $debug
         result['bozo'] = true
         result['bozo_exception'] = feedparser.exc || e 
         use_strict_parser = false
       end
     end
+    
     if not use_strict_parser
       $stderr << "Using LooseFeed\n\n" if $debug
       feedparser = looseklass.new(baseuri, baselang, (known_encoding and 'utf-8' or ''))
       feedparser.parse(data)
     end
+
     result['feed'] = feedparser.feeddata
     result['entries'] = feedparser.entries
     result['version'] = result['version'] || feedparser.version

data/lib/rfeedparser/better_sgmlparser.rb

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 
-class BetterSGMLParserError < Exception; end;
+class BetterSGMLParserError < StandardError; end;
 class BetterSGMLParser < HTML::SGMLParser
   # Replaced Tagfind and Charref Regexps with the ones in feedparser.py
   # This makes things work. 

data/lib/rfeedparser/encoding_helpers.rb

@@ -35,12 +35,14 @@ module FeedParserUtilities
     true_encoding = nil
     
     http_content_type, charset = http_headers['content-type'].to_s.split(';',2)
+
     encoding_regexp = /\s*charset\s*=\s*(?:"|')?(.*?)(?:"|')?\s*$/
     http_encoding = charset.to_s.scan(encoding_regexp).flatten[0]
-    http_encoding = nil if http_encoding.blank?
-      # FIXME Open-Uri returns iso8859-1 if there is no charset header,
-      # but that doesn't pass the tests. Open-Uri claims its following
-      # the right RFC. Are they wrong or do we need to change the tests?
+
+    http_encoding = nil if http_encoding && http_encoding.empty?
+    # FIXME Open-Uri returns iso8859-1 if there is no charset header,
+    # but that doesn't pass the tests. Open-Uri claims its following
+    # the right RFC. Are they wrong or do we need to change the tests?
     
     # Must sniff for non-ASCII-compatible character encodings before
     # searching for XML declaration.  This heuristic is defined in
@@ -224,14 +226,19 @@ unless defined?(Builder::XChar)
       62 => '>'  # right angle bracket
     }
     # http://www.w3.org/TR/REC-xml/#charsets
-    VALID = [[0x9, 0xA, 0xD], (0x20..0xD7FF), 
-    (0xE000..0xFFFD), (0x10000..0x10FFFF)]
+    VALID = [
+      0x9, 0xA, 0xD,
+      (0x20..0xD7FF),
+      (0xE000..0xFFFD),
+      (0x10000..0x10FFFF)
+    ]
   end
 
   class Fixnum
     # xml escaped version of chr
     def xchr
       n = XChar::CP1252[self] || self
+
       case n when *XChar::VALID
         XChar::PREDEFINED[n] or (n<128 ? n.chr : "&##{n};")
       else
@@ -241,11 +248,10 @@ unless defined?(Builder::XChar)
   end
 
   class String
-    alias :old_index :index
-    def to_xs 
+    def to_xs
       unpack('U*').map {|n| n.xchr}.join # ASCII, UTF-8
     rescue
       unpack('C*').map {|n| n.xchr}.join # ISO-8859-1, WIN-1252
     end
   end
-end
+end