rfeedparser 0.9.931 → 0.9.940

data/lib/rfeedparser/feedparserdict.rb

@@ -1,5 +1,5 @@
-#!/usr/bin/ruby
-module FeedParserUtilities
+#!/usr/bin/env ruby
+module FeedParser
   class FeedParserDict < Hash 
 =begin
      The naming of a certain common attribute (such as, "When was the last
@@ -16,26 +16,36 @@ module FeedParserUtilities
      methods check with keymaps to see what attribute the developer "really
      means" if they've asked for one which happens to be in @@keymap's keys.
 =end
-    @@keymap = {'channel' => 'feed',
-	      'items' => 'entries',
-	      'guid' => 'id',
-	      'date' => 'updated',
-	      'date_parsed' => 'updated_parsed',
-	      'description' => ['subtitle', 'summary'],
-	      'url' => ['href'],
-	      'modified' => 'updated',
-	      'modified_parsed' => 'updated_parsed',
-	      'issued' => 'published',
-	      'issued_parsed' => 'published_parsed',
-	      'copyright' => 'rights',
-	      'copyright_detail' => 'rights_detail',
-	      'tagline' => 'subtitle',
-	      'tagline_detail' => 'subtitle_detail'}
-
-    def entries # Apparently, Hash has an entries method!  That blew a good 3 hours or more of my time
-      return self['entries']
+    @@keymap = {
+      'channel' => 'feed',
+      'items' => 'entries',
+	    'guid' => 'id',
+	    'date' => 'updated',
+	    'date_parsed' => 'updated_parsed',
+	    'description' => ['subtitle', 'summary'],
+	    'url' => ['href'],
+	    'modified' => 'updated',
+	    'modified_parsed' => 'updated_parsed',
+	    'issued' => 'published',
+	    'issued_parsed' => 'published_parsed',
+	    'copyright' => 'rights',
+	    'copyright_detail' => 'rights_detail',
+	    'tagline' => 'subtitle',
+	    'tagline_detail' => 'subtitle_detail'
+    }
+    
+    # Apparently, Hash has an entries method!  That blew a good 3 hours or more of my time
+    alias :hash_entries :entries
+    def entries 
+      self['entries']
     end
 
+    # Added to avoid deprecated method wornings
+    alias :object_type :type
+    def type
+      self['type']
+    end
+    
     # We could include the [] rewrite in new using Hash.new's fancy pants block thing
     # but we'd still have to overwrite []= and such. 
     # I'm going to make it easy to turn lists of pairs into FeedParserDicts's though.

data/lib/rfeedparser/forgiving_uri.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 # From Robert Aman's GentleCMS URI.
 # GentleCMS, Copyright (c) 2006 Robert Aman
 #
@@ -535,7 +535,7 @@ class ForgivingURI
     
     # Merges two URIs together.
     def merge(uri)
-      return self + uri
+      return (self + uri)
     end
     
     # Destructive form of merge.
@@ -940,12 +940,14 @@ class ForgivingURI
 def urljoin(base, uri)
   urifixer = /^([A-Za-z][A-Za-z0-9+-.]*:\/\/)(\/*)(.*?)/u
   uri = uri.sub(urifixer, '\1\3') 
-  begin
-    return ForgivingURI.join(base, uri).to_s 
-  rescue URI::BadURIError => e
-    if ForgivingURI.parse(base).relative? 
-      return ForgivingURI.parse(uri).to_s
+  pbase = ForgivingURI.parse(base) rescue nil
+  if pbase && pbase.absolute?
+    puri = ForgivingURI.parse(uri) rescue nil
+    if puri && puri.relative?
+      # ForgivingURI.join does the wrong thing.  What the hell.
+      return ForgivingURI.join(base, uri).to_s.gsub(/[^:]\/{2,}/, '')
     end
   end
+  return uri
 end
 

data/lib/rfeedparser/markup_helpers.rb

@@ -1,14 +1,11 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 module FeedParserUtilities
-  #FIXME we need to find a better place for this method
   def stripDoctype(data)
-=begin
-Strips DOCTYPE from XML document, returns (rss_version, stripped_data)
-
-    rss_version may be 'rss091n' or None
-    stripped_data is the same XML document, minus the DOCTYPE
-=end
+    #Strips DOCTYPE from XML document, returns (rss_version, stripped_data)
+    #rss_version may be 'rss091n' or None
+    #stripped_data is the same XML document, minus the DOCTYPE
     entity_pattern = /<!ENTITY(.*?)>/m # m is for Regexp::MULTILINE
+    
     data = data.gsub(entity_pattern,'')
 
     doctype_pattern = /<!DOCTYPE(.*?)>/m
@@ -27,7 +24,7 @@ Strips DOCTYPE from XML document, returns (rss_version, stripped_data)
     data = data.sub(doctype_pattern, '')
     return version, data
   end
-
+  
   def resolveRelativeURIs(htmlSource, baseURI, encoding)
     $stderr << "entering resolveRelativeURIs\n" if $debug # FIXME write a decent logger
     relative_uris = [ ['a','href'],
@@ -60,11 +57,11 @@ Strips DOCTYPE from XML document, returns (rss_version, stripped_data)
     relative_uris.each do |l|
       ename, eattr = l
       h.search(ename).each do |elem|
-	euri = elem.attributes[eattr]
-        # FIXME uses the URI.encode method.  should it?
-	if euri and not euri.empty? and ForgivingURI.parse(URI.encode(euri)).relative? 
-	  elem.attributes[eattr] = urljoin(baseURI, euri)
-	end
+        euri = elem.attributes[eattr]
+        uri = ForgivingURI.parse(URI.encode(euri)) rescue nil
+        if euri and not euri.empty? and uri and uri.relative?
+          elem.raw_attributes[eattr] = urljoin(baseURI, euri)
+        end
       end
     end
     return h.to_html

data/lib/rfeedparser/parser_mixin.rb

@@ -1,5 +1,7 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
+module FeedParser
 module FeedParserMixin
+  include FeedParserUtilities
   attr_accessor :feeddata, :version, :namespacesInUse, :date_handlers
 
   def startup(baseuri=nil, baselang=nil, encoding='utf-8')
@@ -103,10 +105,6 @@ module FeedParserMixin
     if baselang 
       @feeddata['language'] = baselang.gsub('_','-')
     end
-    @date_handlers = [:_parse_date_rfc822,
-      :_parse_date_hungarian, :_parse_date_greek,:_parse_date_mssql,
-      :_parse_date_nate,:_parse_date_onblog,:_parse_date_w3dtf,:_parse_date_iso8601
-    ]
     $stderr << "Leaving startup\n" if $debug # My addition
   end
 
@@ -873,7 +871,9 @@ module FeedParserMixin
 
   def _end_published
     value = pop('published')
-    _save('published_parsed', parse_date(value))
+    d = parse_date(value)
+    _save('published_parsed', extract_tuple(d))
+    _save('published_time', d)
   end
   alias :_end_dcterms_issued :_end_published
   alias :_end_issued :_end_published
@@ -888,7 +888,9 @@ module FeedParserMixin
 
   def _end_updated
     value = pop('updated')
-    _save('updated_parsed', parse_date(value))
+    d = parse_date(value)
+    _save('updated_parsed', extract_tuple(d))
+    _save('updated_time', d)
   end
   alias :_end_modified :_end_updated
   alias :_end_dcterms_modified :_end_updated
@@ -902,7 +904,9 @@ module FeedParserMixin
 
   def _end_created
     value = pop('created')
-    _save('created_parsed', parse_date(value))
+    d = parse_date(value)
+    _save('created_parsed', extract_tuple(d))
+    _save('created_time', d)
   end
   alias :_end_dcterms_created :_end_created
 
@@ -910,7 +914,9 @@ module FeedParserMixin
     push('expired', true)
   end
   def _end_expirationdate
-    _save('expired_parsed', parse_date(pop('expired')))
+    d = parse_date(pop('expired'))
+    _save('expired_parsed', extract_tuple(d))
+    _save('expired_time', d)
   end
 
   def _start_cc_license(attrsD)
@@ -1234,5 +1240,4 @@ module FeedParserMixin
   end
   
 end # End FeedParserMixin
-
-
+end

data/lib/rfeedparser/parsers.rb

@@ -1,5 +1,4 @@
-#!/usr/bin/ruby
-
+#!/usr/bin/env ruby
 
 module FeedParser
   class StrictFeedParser < XML::SAX::HandlerBase # expat

data/lib/rfeedparser/scrub.rb

@@ -1,47 +1,45 @@
-#!/usr/bin/ruby
-gem 'hpricot', "=0.5"
-require 'hpricot'
+#!/usr/bin/env ruby
 # This used to be based on Michael Moen's Hpricot#scrub, but that seems to 
 # have only been part of its evolution. Hpricot#scrub is cool code, though.
 # http://underpantsgnome.com/2007/01/20/hpricot-scrub
 module Hpricot
   Acceptable_Elements = ['a', 'abbr', 'acronym', 'address', 'area', 'b',
-      'big', 'blockquote', 'br', 'button', 'caption', 'center', 'cite',
-      'code', 'col', 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt',
-      'em', 'fieldset', 'font', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
-      'hr', 'i', 'img', 'input', 'ins', 'kbd', 'label', 'legend', 'li', 'map',
-      'menu', 'ol', 'optgroup', 'option', 'p', 'pre', 'q', 's', 'samp',
-      'select', 'small', 'span', 'strike', 'strong', 'sub', 'sup', 'table',
-      'tbody', 'td', 'textarea', 'tfoot', 'th', 'thead', 'tr', 'tt', 'u',
-      'ul', 'var'
+    'big', 'blockquote', 'br', 'button', 'caption', 'center', 'cite',
+    'code', 'col', 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt',
+    'em', 'fieldset', 'font', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+    'hr', 'i', 'img', 'input', 'ins', 'kbd', 'label', 'legend', 'li', 'map',
+    'menu', 'ol', 'optgroup', 'option', 'p', 'pre', 'q', 's', 'samp',
+    'select', 'small', 'span', 'strike', 'strong', 'sub', 'sup', 'table',
+    'tbody', 'td', 'textarea', 'tfoot', 'th', 'thead', 'tr', 'tt', 'u',
+    'ul', 'var'
   ]
 
   Acceptable_Attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
-      'action', 'align', 'alt', 'axis', 'border', 'cellpadding',
-      'cellspacing', 'char', 'charoff', 'charset', 'checked', 'cite', 'class',
-      'clear', 'cols', 'colspan', 'color', 'compact', 'coords', 'datetime',
-      'dir', 'disabled', 'enctype', 'for', 'frame', 'headers', 'height',
-      'href', 'hreflang', 'hspace', 'id', 'ismap', 'label', 'lang',
-      'longdesc', 'maxlength', 'media', 'method', 'multiple', 'name',
-      'nohref', 'noshade', 'nowrap', 'prompt', 'readonly', 'rel', 'rev',
-      'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size',
-      'span', 'src', 'start', 'summary', 'tabindex', 'target', 'title', 
-      'type', 'usemap', 'valign', 'value', 'vspace', 'width', 'xml:lang'
+    'action', 'align', 'alt', 'axis', 'border', 'cellpadding',
+    'cellspacing', 'char', 'charoff', 'charset', 'checked', 'cite', 'class',
+    'clear', 'cols', 'colspan', 'color', 'compact', 'coords', 'datetime',
+    'dir', 'disabled', 'enctype', 'for', 'frame', 'headers', 'height',
+    'href', 'hreflang', 'hspace', 'id', 'ismap', 'label', 'lang',
+    'longdesc', 'maxlength', 'media', 'method', 'multiple', 'name',
+    'nohref', 'noshade', 'nowrap', 'prompt', 'readonly', 'rel', 'rev',
+    'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size',
+    'span', 'src', 'start', 'summary', 'tabindex', 'target', 'title', 
+    'type', 'usemap', 'valign', 'value', 'vspace', 'width', 'xml:lang'
   ]
 
   Unacceptable_Elements_With_End_Tag = ['script', 'applet']
 
   Acceptable_Css_Properties = ['azimuth', 'background-color',
-      'border-bottom-color', 'border-collapse', 'border-color',
-      'border-left-color', 'border-right-color', 'border-top-color', 'clear',
-      'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font',
-      'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight',
-      'height', 'letter-spacing', 'line-height', 'overflow', 'pause',
-      'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness',
-      'speak', 'speak-header', 'speak-numeral', 'speak-punctuation',
-      'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent',
-      'unicode-bidi', 'vertical-align', 'voice-family', 'volume',
-      'white-space', 'width'
+    'border-bottom-color', 'border-collapse', 'border-color',
+    'border-left-color', 'border-right-color', 'border-top-color', 'clear',
+    'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font',
+    'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight',
+    'height', 'letter-spacing', 'line-height', 'overflow', 'pause',
+    'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness',
+    'speak', 'speak-header', 'speak-numeral', 'speak-punctuation',
+    'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent',
+    'unicode-bidi', 'vertical-align', 'voice-family', 'volume',
+    'white-space', 'width'
   ]
 
   # survey of common keywords found in feeds
@@ -82,38 +80,38 @@ module Hpricot
 
   # svgtiny + class + opacity + offset + xmlns + xmlns:xlink
   Svg_Attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic',
-     'arabic-form', 'ascent', 'attributeName', 'attributeType',
-     'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height',
-     'class', 'color', 'color-rendering', 'content', 'cx', 'cy', 'd',
-     'descent', 'display', 'dur', 'end', 'fill', 'fill-rule', 'font-family',
-     'font-size', 'font-stretch', 'font-style', 'font-variant',
-     'font-weight', 'from', 'fx', 'fy', 'g1', 'g2', 'glyph-name', 
-     'gradientUnits', 'hanging', 'height', 'horiz-adv-x', 'horiz-origin-x',
-     'id', 'ideographic', 'k', 'keyPoints', 'keySplines', 'keyTimes',
-     'lang', 'mathematical', 'max', 'min', 'name', 'offset', 'opacity',
-     'origin', 'overline-position', 'overline-thickness', 'panose-1',
-     'path', 'pathLength', 'points', 'preserveAspectRatio', 'r',
-     'repeatCount', 'repeatDur', 'requiredExtensions', 'requiredFeatures',
-     'restart', 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 
-     'stop-color', 'stop-opacity', 'strikethrough-position',
-     'strikethrough-thickness', 'stroke', 'stroke-dasharray',
-     'stroke-dashoffset', 'stroke-linecap', 'stroke-linejoin',
-     'stroke-miterlimit', 'stroke-width', 'systemLanguage', 'target',
-     'text-anchor', 'to', 'transform', 'type', 'u1', 'u2',
-     'underline-position', 'underline-thickness', 'unicode',
-     'unicode-range', 'units-per-em', 'values', 'version', 'viewBox',
-     'visibility', 'width', 'widths', 'x', 'x-height', 'x1', 'x2',
-     'xlink:actuate', 'xlink:arcrole', 'xlink:href', 'xlink:role',
-     'xlink:show', 'xlink:title', 'xlink:type', 'xml:base', 'xml:lang',
-     'xml:space', 'xmlns', 'xmlns:xlink', 'y', 'y1', 'y2', 'zoomAndPan'
+    'arabic-form', 'ascent', 'attributeName', 'attributeType',
+    'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height',
+    'class', 'color', 'color-rendering', 'content', 'cx', 'cy', 'd',
+    'descent', 'display', 'dur', 'end', 'fill', 'fill-rule', 'font-family',
+    'font-size', 'font-stretch', 'font-style', 'font-variant',
+    'font-weight', 'from', 'fx', 'fy', 'g1', 'g2', 'glyph-name', 
+    'gradientUnits', 'hanging', 'height', 'horiz-adv-x', 'horiz-origin-x',
+    'id', 'ideographic', 'k', 'keyPoints', 'keySplines', 'keyTimes',
+    'lang', 'mathematical', 'max', 'min', 'name', 'offset', 'opacity',
+    'origin', 'overline-position', 'overline-thickness', 'panose-1',
+    'path', 'pathLength', 'points', 'preserveAspectRatio', 'r',
+    'repeatCount', 'repeatDur', 'requiredExtensions', 'requiredFeatures',
+    'restart', 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 
+    'stop-color', 'stop-opacity', 'strikethrough-position',
+    'strikethrough-thickness', 'stroke', 'stroke-dasharray',
+    'stroke-dashoffset', 'stroke-linecap', 'stroke-linejoin',
+    'stroke-miterlimit', 'stroke-width', 'systemLanguage', 'target',
+    'text-anchor', 'to', 'transform', 'type', 'u1', 'u2',
+    'underline-position', 'underline-thickness', 'unicode',
+    'unicode-range', 'units-per-em', 'values', 'version', 'viewBox',
+    'visibility', 'width', 'widths', 'x', 'x-height', 'x1', 'x2',
+    'xlink:actuate', 'xlink:arcrole', 'xlink:href', 'xlink:role',
+    'xlink:show', 'xlink:title', 'xlink:type', 'xml:base', 'xml:lang',
+    'xml:space', 'xmlns', 'xmlns:xlink', 'y', 'y1', 'y2', 'zoomAndPan'
   ]
 
   Svg_Attr_Map = nil
   Svg_Elem_Map = nil
 
   Acceptable_Svg_Properties = [ 'fill', 'fill-opacity', 'fill-rule',
-      'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin',
-      'stroke-opacity'
+    'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin',
+    'stroke-opacity'
   ]
 
   unless $compatible 
@@ -148,11 +146,9 @@ module Hpricot
   class Elem
     def strip_attributes
       unless attributes.nil?
-	attributes.each do |atr|
-	  unless Acceptable_Attributes.include?atr[0] 
-	    remove_attribute(atr[0]) 
-	  end
-	end
+        ra = {}
+        raw_attributes.keys.each{|atr| ra[atr] = raw_attributes[atr] if Acceptable_Attributes.include?(atr) }
+        self.raw_attributes = ra
       end
     end
   end
@@ -160,35 +156,44 @@ end
 
 module FeedParserUtilities
   class SanitizerDoc < Hpricot::Doc
-
+    
     def scrub
-      traverse_all_element do |e| 
-	if e.elem? 
-	  if Acceptable_Elements.include?e.name
-	    e.strip_attributes
-	  else
-	    if Unacceptable_Elements_With_End_Tag.include?e.name
-	      e.inner_html = ''
-	    end
-	    e.swap(SanitizerDoc.new(e.children).scrub.to_html) # The important part
-	  end
-	elsif e.doctype?
-	  e.parent.children.delete(e)
-	elsif e.text?
-	  ets = e.to_s
-	  ets.gsub!(/&#39;/, "'") 
-	    ets.gsub!(/&#34;/, '"')
-	    ets.gsub!(/\r/,'')
-	  e.swap(ets)
-	else
-	end
+      others = children.map do |e|
+        if e.elem?
+          if Acceptable_Elements.include?e.name
+            e.strip_attributes
+            e.inner_html = SanitizerDoc.new(e.children).scrub
+            result = e
+          else
+            result = e
+            
+            if Unacceptable_Elements_With_End_Tag.include?e.name
+              result = nil
+            end
+            
+            if result 
+              result = SanitizerDoc.new(result.children).scrub   # The important part
+            end            
+          end
+          
+        elsif e.doctype?
+          result = nil
+
+        elsif e.text?
+          ets = e.to_html
+          ets.gsub!(/&#39;/, "'") 
+          ets.gsub!(/&#34;/, '"')
+          ets.gsub!(/\r/,'')
+          result = ets
+        end
+        result
       end
-
+      
       unless $compatible # FIXME nonworking
-	# yes, that '/' should be there. It's a search method. See the Hpricot docs.
-	(self/tag).strip_style(@config[:allow_css_properties], @config[:allow_css_keywords])
+        # yes, that '/' should be there. It's a search method. See the Hpricot docs.
+        (self/tag).strip_style(@config[:allow_css_properties], @config[:allow_css_keywords])
       end
-      return self
+      return others.compact.join
     end
   end
 
@@ -200,8 +205,8 @@ module FeedParserUtilities
   def sanitizeHTML(html,encoding)
     # FIXME Tidy not yet supported
     html = html.gsub(/<!((?!DOCTYPE|--|\[))/, '&lt;!\1')
-      h = SanitizerDoc(html)
-      h = h.scrub
-      return h.to_html.strip
+    h = SanitizerDoc(html)
+    h = h.scrub
+    return h.strip
   end
 end