rfeedparser 0.9.940 → 0.9.950

data/lib/rfeedparser/expat_parser.rb

@@ -0,0 +1,159 @@
+# If you don't have libxml installed (or for some reason you would
+# rather use expat) you can fall back upon this for your parser. Be
+# advised that the bindings for expat are much older (from the days of
+# Ruby 1.6.7) and are unmaintained. If you want to force use of expat
+# even when libxml is installed, you can do this:
+#
+#   feed = FeedParser.parse("some-feed-stream-filepath-or-url",
+#                           :strict => FeedParser::Expat::StrictFeedParser)
+
+require 'xml/saxdriver'
+
+module FeedParser
+ module Expat
+   class StrictFeedParser
+     attr_reader :handler
+     def initialize(baseuri, baselang)
+       @handler = StrictFeedParserHandler.new(baseuri, baselang, 'utf-8')
+     end
+
+     def parse(data)
+       # initialize the SAX parser
+       saxparser = XML::SAX::Helpers::ParserFactory.makeParser("XML::Parser::SAXDriver")
+       saxparser.setDocumentHandler(@handler)
+       saxparser.setDTDHandler(@handler)
+       saxparser.setEntityResolver(@handler)
+       saxparser.setErrorHandler(@handler)
+
+       inputdata = XML::SAX::InputSource.new('parsedfeed')
+       inputdata.setByteStream(StringIO.new(data))
+       begin
+         saxparser.parse(inputdata)
+       rescue XML::SAX::SAXParseException => err
+         # This does not inherit from StandardError as it should, so
+         # we have to catch and re-raise specially.
+         raise err.to_s
+       end
+     end
+   end
+
+   class StrictFeedParserHandler < XML::SAX::HandlerBase # expat
+     include FeedParserMixin
+
+     attr_accessor :bozo, :entries, :feeddata, :exc
+     def initialize(baseuri, baselang, encoding)
+       $stderr << "trying StrictFeedParser\n" if $debug
+       startup(baseuri, baselang, encoding)
+       @bozo = false
+       @exc = nil
+       super()
+     end
+
+     def getPos
+       [@locator.getSystemId, @locator.getLineNumber]
+     end
+
+     def getAttrs(attrs)
+       ret = []
+       for i in 0..attrs.getLength
+   ret.push([attrs.getName(i), attrs.getValue(i)])
+       end
+       ret
+     end
+
+     def setDocumentLocator(loc)
+       @locator = loc
+     end
+
+     def startDoctypeDecl(name, pub_sys, long_name, uri)
+       #Nothing is done here. What could we do that is neat and useful?
+     end
+
+     def startNamespaceDecl(prefix, uri)
+       trackNamespace(prefix, uri)
+     end
+
+     def endNamespaceDecl(prefix)
+     end
+
+     def startElement(name, attrs)
+       name =~ /^(([^;]*);)?(.+)$/ # Snag namespaceuri from name
+   namespaceuri = ($2 || '').downcase
+       name = $3
+       if /backend\.userland\.com\/rss/ =~ namespaceuri
+   # match any backend.userland.com namespace
+   namespaceuri = 'http://backend.userland.com/rss'
+       end
+       prefix = @matchnamespaces[namespaceuri]
+       # No need to raise UndeclaredNamespace, Expat does that for us with
+       "unbound prefix (XMLParserError)"
+       if prefix and not prefix.empty?
+   name = prefix + ':' + name
+       end
+       name.downcase!
+       unknown_starttag(name, attrs)
+     end
+
+     def character(text, start, length)
+       #handle_data(CGI.unescapeHTML(text))
+       handle_data(text)
+     end
+     # expat provides "character" not "characters"!
+     alias :characters :character # Just in case.
+
+     def startCdata(content)
+       handle_data(content)
+     end
+
+     def endElement(name)
+       name =~ /^(([^;]*);)?(.+)$/ # Snag namespaceuri from name
+   namespaceuri = ($2 || '').downcase
+       prefix = @matchnamespaces[namespaceuri]
+       if prefix and not prefix.empty?
+   localname = prefix + ':' + name
+       end
+       name.downcase!
+       unknown_endtag(name)
+     end
+
+     def comment(comment)
+       handle_comment(comment)
+     end
+
+     def entityDecl(*foo)
+     end
+
+     def unparsedEntityDecl(*foo)
+     end
+
+     def error(exc)
+       @bozo = true
+       @exc = exc
+     end
+
+     def fatalError(exc)
+       error(exc)
+       raise exc
+     end
+   end
+ end
+end
+
+
+# FIXME line 5 maps to line 171 in saxdriver.rb. note that there is no return 
+# in the original
+class XML::Parser::SAXDriver
+   def openInputStream(stream)
+      if stream.getByteStream
+        return stream
+      else stream.getSystemId
+        url = URL.new(stream.getSystemId)
+        if url.scheme == 'file' && url.login == 'localhost'
+          s = open(url.urlpath)
+          stream.setByteStream(s)
+          return stream
+        end
+      end
+      return nil
+    end
+end

data/lib/rfeedparser/libxml_parser.rb

@@ -0,0 +1,76 @@
+gem 'libxml-ruby', '>=0.8.3'
+require 'libxml'
+
+module FeedParser
+ module LibXML
+   class StrictFeedParser
+     attr_reader :handler
+
+     def initialize(baseuri, baselang)
+       @handler = StrictFeedParserHandler.new(baseuri, baselang, 'utf-8')
+     end
+
+     def parse(data)
+       saxparser = LibXML::XML::SaxParser.new
+       saxparser.callbacks = @handler
+       saxparser.string = data
+       saxparser.parse
+     end
+   end
+
+   class StrictFeedParserHandler
+     include LibXML::XML::SaxParser::Callbacks
+     include FeedParserMixin
+
+     attr_accessor :bozo, :entries, :feeddata, :exc
+     def initialize(baseuri, baselang, encoding)
+       $stderr.puts "trying LibXML::StrictFeedParser" if $debug
+       startup(baseuri, baselang, encoding)
+       @bozo = false
+     end
+
+     def on_start_element(name, attrs)
+       name =~ /^(([^;]*);)?(.+)$/ # Snag namespaceuri from name
+       namespaceuri = ($2 || '').downcase
+       name = $3
+       if /backend\.userland\.com\/rss/ =~ namespaceuri
+         # match any backend.userland.com namespace
+         namespaceuri = 'http://backend.userland.com/rss'
+       end
+       prefix = @matchnamespaces[namespaceuri]
+
+       if prefix and not prefix.empty?
+         name = prefix + ':' + name
+       end
+
+       name.downcase!
+       unknown_starttag(name, attrs)
+     end
+
+     def on_characters(text)
+       handle_data(text)
+     end
+
+     def on_cdata_block(content)
+       handle_data(content)
+     end
+
+     def on_end_element(name)
+       name =~ /^(([^;]*);)?(.+)$/ # Snag namespaceuri from name
+       namespaceuri = ($2 || '').downcase
+       prefix = @matchnamespaces[namespaceuri]
+       if prefix and not prefix.empty?
+         localname = prefix + ':' + name
+       end
+       name.downcase!
+       unknown_endtag(name)
+     end
+
+     def on_parser_error(exc)
+       @bozo = true
+       @exc = exc
+       raise exc
+     end
+   end
+ end
+end

data/lib/rfeedparser/loose_feed_parser.rb

@@ -0,0 +1,75 @@
+module FeedParser
+ class LooseFeedParser < BetterSGMLParser
+   include FeedParserMixin
+   # We write the methods that were in BaseHTMLProcessor in the python code
+   # in here directly. We do this because if we inherited from
+   # BaseHTMLProcessor but then included from FeedParserMixin, the methods
+   # of Mixin would overwrite the methods we inherited from
+   # BaseHTMLProcessor. This is exactly the opposite of what we want to
+   # happen!
+
+   attr_accessor :encoding, :bozo, :feeddata, :entries, :namespacesInUse
+
+   Elements_No_End_Tag = ['area', 'base', 'basefont', 'br', 'col', 'frame', 'hr',
+     'img', 'input', 'isindex', 'link', 'meta', 'param']
+   New_Declname_Re = /[a-zA-Z][-_.a-zA-Z0-9:]*\s*/
+     alias :sgml_feed :feed # feed needs to mapped to feeddata, not the SGMLParser method feed. I think.
+   def feed
+     @feeddata
+   end
+   def feed=(data)
+     @feeddata = data
+   end
+
+   def initialize(baseuri, baselang, encoding)
+     startup(baseuri, baselang, encoding)
+     super() # Keep the parentheses! No touchy.
+   end
+
+   def reset
+     @pieces = []
+     super
+   end
+
+   def parse(data)
+     data.gsub!(/<!((?!DOCTYPE|--|\[))/i,  '&lt;!\1')
+ data.gsub!(/<([^<\s]+?)\s*\/>/) do |tag|
+   clean = tag[1..-3].strip
+   if Elements_No_End_Tag.include?clean
+     tag
+   else
+   '<'+clean+'></'+clean+'>'
+   end
+ end
+
+ data.gsub!(/&#39;/, "'")
+   data.gsub!(/&#34;/, "'")
+   if @encoding and not @encoding.empty? # FIXME unicode check type(u'')
+     data = uconvert(data,'utf-8',@encoding)
+   end
+ sgml_feed(data) # see the alias above
+   end
+
+
+   def decodeEntities(element, data)
+     data.gsub!('&#60;', '&lt;')
+     data.gsub!('&#x3c;', '&lt;')
+     data.gsub!('&#62;', '&gt;')
+     data.gsub!('&#x3e;', '&gt;')
+     data.gsub!('&#38;', '&amp;')
+     data.gsub!('&#x26;', '&amp;')
+     data.gsub!('&#34;', '&quot;')
+     data.gsub!('&#x22;', '&quot;')
+     data.gsub!('&#39;', '&apos;')
+     data.gsub!('&#x27;', '&apos;')
+     if @contentparams.has_key? 'type' and not ((@contentparams['type'] || 'xml') =~ /xml$/u)
+ data.gsub!('&lt;', '<')
+ data.gsub!('&gt;', '>')
+ data.gsub!('&amp;', '&')
+ data.gsub!('&quot;', '"')
+ data.gsub!('&apos;', "'")
+     end
+     return data
+   end
+ end
+end

data/lib/rfeedparser/markup_helpers.rb

@@ -58,7 +58,7 @@ module FeedParserUtilities
       ename, eattr = l
       h.search(ename).each do |elem|
         euri = elem.attributes[eattr]
-        uri = ForgivingURI.parse(URI.encode(euri)) rescue nil
+        uri = Addressable::URI.parse(Addressable::URI.encode(euri)) rescue nil
         if euri and not euri.empty? and uri and uri.relative?
           elem.raw_attributes[eattr] = urljoin(baseURI, euri)
         end

data/lib/rfeedparser/monkey_patches.rb

@@ -0,0 +1,10 @@
+class Time
+  class << self
+    COMMON_YEAR_DAYS_IN_MONTH = [nil, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+
+    def days_in_month(month, year = now.year)
+      return 29 if month == 2 && ::Date.gregorian_leap?(year)
+      COMMON_YEAR_DAYS_IN_MONTH[month]
+    end
+  end
+end

data/lib/rfeedparser/parser_mixin.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+
 module FeedParser
 module FeedParserMixin
   include FeedParserUtilities
@@ -102,6 +103,7 @@ module FeedParserMixin
     @langstack = []
     @baseuri = baseuri || ''
     @lang = baselang || nil
+    @has_title = false
     if baselang 
       @feeddata['language'] = baselang.gsub('_','-')
     end
@@ -136,6 +138,7 @@ module FeedParserMixin
       # if no xml:lang is specified, use parent lang
       lang = @lang
     end
+
     if lang and not lang.empty? # Seriously, this cannot be correct
       if ['feed', 'rss', 'rdf:RDF'].include?tag
         @feeddata['language'] = lang.gsub('_','-')
@@ -287,7 +290,7 @@ module FeedParserMixin
     # not containing any character or entity references
     return if @elementstack.nil? or @elementstack.empty?
     if escape and @contentparams['type'] == 'application/xhtml+xml'
-      text = text.to_xs 
+      text = text.to_xs
     end
     @elementstack[-1][2] << text
   end
@@ -367,6 +370,7 @@ module FeedParserMixin
     return if @elementstack.nil? or @elementstack.empty?
     return if @elementstack[-1][0] != element
     element, expectingText, pieces = @elementstack.pop
+
     if pieces.class == Array
       output = pieces.join('')
     else
@@ -386,7 +390,7 @@ module FeedParserMixin
     end
 
     # resolve relative URIs
-    if @can_be_relative_uri.include?element and output and not output.empty?
+    if @can_be_relative_uri.include?(element) && output && !output.empty?
       output = resolveURI(output)
     end
 
@@ -400,26 +404,28 @@ module FeedParserMixin
     @contentparams.delete('base64')
 
     # resolve relative URIs within embedded markup
-    if @html_types.include?mapContentType(@contentparams['type'] || 'text/html')
-      if @can_contain_relative_uris.include?element
+    if @html_types.include?(mapContentType(@contentparams['type'] || 'text/html'))
+      if @can_contain_relative_uris.include?(element)
         output = FeedParser.resolveRelativeURIs(output, @baseuri, @encoding)
       end
     end
     # sanitize embedded markup
-    if @html_types.include?mapContentType(@contentparams['type'] || 'text/html')
-      if @can_contain_dangerous_markup.include?element
+    if @html_types.include?(mapContentType(@contentparams['type'] || 'text/html'))
+      if @can_contain_dangerous_markup.include?(element)
         output = FeedParser.sanitizeHTML(output, @encoding)
       end
     end
 
     if @encoding and not @encoding.empty? and @encoding != 'utf-8'
-      output = uconvert(output, @encoding, 'utf-8') 
+      output = uconvert(output, @encoding, 'utf-8')
       # FIXME I turn everything into utf-8, not unicode, originally because REXML was being used but now beause I haven't tested it out yet.
     end
 
     # categories/tags/keywords/whatever are handled in _end_category
     return output if element == 'category'
 
+    return output if element == 'title' and @has_title
+
     # store output in appropriate place(s)
     if @inentry and not @insource
       if element == 'content'
@@ -453,6 +459,7 @@ module FeedParserMixin
         context[element + '_detail'] = contentparams
       end
     end
+
     return output
   end
 
@@ -578,6 +585,7 @@ module FeedParserMixin
 
   def _start_image(attrsD)
     @inimage = true
+    @has_title = false
     push('image', false)
     context = getContext()
     context['image'] ||= FeedParserDict.new
@@ -590,6 +598,7 @@ module FeedParserMixin
 
   def _start_textinput(attrsD)
     @intextinput = true
+    @has_title = false
     push('textinput', false)
     context = getContext()
     context['textinput'] ||= FeedParserDict.new
@@ -825,6 +834,7 @@ module FeedParserMixin
     @entries << FeedParserDict.new
     push('item', false)
     @inentry = true
+    @has_title = false
     @guidislink = false
     id = getAttribute(attrsD, 'rdf:about')
     if id and not id.empty?
@@ -1052,15 +1062,21 @@ module FeedParserMixin
 
   def _end_title
     value = popContent('title')
-    context = getContext()
+    context = getContext
     if @intextinput
       context['textinput']['title'] = value
     elsif @inimage
       context['image']['title'] = value
     end
+    @has_title = true
   end
   alias :_end_dc_title :_end_title
-  alias :_end_media_title :_end_title
+
+  def _end_media_title
+    orig_has_title = @has_title
+    _end_title
+    @has_title = orig_has_title
+  end
 
   def _start_description(attrsD)
     context = getContext()
@@ -1179,6 +1195,7 @@ module FeedParserMixin
 
   def _start_source(attrsD)
     @insource = true
+    @has_title = false
   end
 
   def _end_source
@@ -1240,4 +1257,18 @@ module FeedParserMixin
   end
   
 end # End FeedParserMixin
-end
+end
+
+def urljoin(base, uri)
+  urifixer = /^([A-Za-z][A-Za-z0-9+-.]*:\/\/)(\/*)(.*?)/u
+  uri = uri.sub(urifixer, '\1\3') 
+  pbase = Addressable::URI.parse(base) rescue nil
+  if pbase && pbase.absolute?
+    puri = Addressable::URI.parse(uri) rescue nil
+    if puri && puri.relative?
+      # ForgivingURI.join does the wrong thing.  What the hell.
+      return Addressable::URI.join(base, uri).to_s.gsub(/[^:]\/{2,}/, '')
+    end
+  end
+  return uri
+end