penso-feedparser 0.8

data/lib/feedparser/rexml_patch.rb

@@ -0,0 +1,28 @@
+require 'feedparser/textconverters'
+
+# Patch for REXML
+# Very ugly patch to make REXML error-proof.
+# The problem is REXML uses IConv, which isn't error-proof at all.
+# With those changes, it uses unpack/pack with some error handling
+module REXML
+  module Encoding
+    def decode(str)
+      return str.toUTF8(@encoding)
+    end
+
+    def encode(str)
+      return str
+    end
+
+    def encoding=(enc)
+      return if defined? @encoding and enc == @encoding
+      @encoding = enc || 'utf-8'
+    end
+  end
+
+  class Element
+    def children
+      @children
+    end
+  end
+end

data/lib/feedparser/sgml-parser.rb

@@ -0,0 +1,332 @@
+# A parser for SGML, using the derived class as static DTD.
+# from http://raa.ruby-lang.org/project/html-parser
+module FeedParser
+  class SGMLParser
+    # Regular expressions used for parsing:
+    Interesting = /[&<]/
+    Incomplete = Regexp.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|' +
+                                '<([a-zA-Z][^<>]*|/([a-zA-Z][^<>]*)?|' +
+                                '![^<>]*)?')
+
+    Entityref = /&([a-zA-Z][-.a-zA-Z0-9]*);/
+    Charref = /&#([0-9]+);/
+
+    Starttagopen = /<[>a-zA-Z]/
+    Endtagopen = /<\/[<>a-zA-Z]/
+    Endbracket = /[<>]/
+    Special = /<![^<>]*>/
+    Commentopen = /<!--/
+    Commentclose = /--[ \t\n]*>/
+    Tagfind = /[a-zA-Z][a-zA-Z0-9.-]*/
+    Attrfind = Regexp.compile('[\s,]*([a-zA-Z_][a-zA-Z_0-9.-]*)' +
+                              '(\s*=\s*' +
+                              "('[^']*'" +
+                              '|"[^"]*"' +
+                              '|[-~a-zA-Z0-9,./:+*%?!()_#=]*))?')
+
+    Entitydefs =
+      {'lt'=>'<', 'gt'=>'>', 'amp'=>'&', 'quot'=>'"', 'apos'=>'\''}
+
+    def initialize(verbose=false)
+      @verbose = verbose
+      reset
+    end
+
+    def reset
+      @rawdata = ''
+      @stack = []
+      @lasttag = '???'
+      @nomoretags = false
+      @literal = false
+    end
+
+    def has_context(gi)
+      @stack.include? gi
+    end
+
+    def setnomoretags
+      @nomoretags = true
+      @literal = true
+    end
+
+    def setliteral(*args)
+      @literal = true
+    end
+
+    def feed(data)
+      @rawdata << data
+      goahead(false)
+    end
+
+    def close
+      goahead(true)
+    end
+
+    def goahead(_end)
+      rawdata = @rawdata
+      i = 0
+      n = rawdata.length
+      while i < n
+        if @nomoretags
+          handle_data(rawdata[i..(n-1)])
+          i = n
+          break
+        end
+        j = rawdata.index(Interesting, i)
+        j = n unless j
+        if i < j
+          handle_data(rawdata[i..(j-1)])
+        end
+        i = j
+        break if (i == n)
+        if rawdata[i] == ?< #
+          if rawdata.index(Starttagopen, i) == i
+            if @literal
+              handle_data(rawdata[i, 1])
+              i += 1
+              next
+            end
+            k = parse_starttag(i)
+            break unless k
+            i = k
+            next
+          end
+          if rawdata.index(Endtagopen, i) == i
+            k = parse_endtag(i)
+            break unless k
+            i = k
+            @literal = false
+            next
+          end
+          if rawdata.index(Commentopen, i) == i
+            if @literal
+              handle_data(rawdata[i,1])
+              i += 1
+              next
+            end
+            k = parse_comment(i)
+            break unless k
+            i += k
+            next
+          end
+          if rawdata.index(Special, i) == i
+            if @literal
+              handle_data(rawdata[i, 1])
+              i += 1
+              next
+            end
+            k = parse_special(i)
+            break unless k
+            i += k
+            next
+          end
+        elsif rawdata[i] == ?& #
+          if rawdata.index(Charref, i) == i
+            i += $&.length
+            handle_charref($1)
+            i -= 1 unless rawdata[i-1] == ?;
+            next
+          end
+          if rawdata.index(Entityref, i) == i
+            i += $&.length
+            handle_entityref($1)
+            i -= 1 unless rawdata[i-1] == ?;
+            next
+          end
+        else
+          raise RuntimeError, 'neither < nor & ??'
+        end
+        # We get here only if incomplete matches but
+        # nothing else
+        match = rawdata.index(Incomplete, i)
+        unless match == i
+          handle_data(rawdata[i, 1])
+          i += 1
+          next
+        end
+        j = match + $&.length
+        break if j == n # Really incomplete
+        handle_data(rawdata[i..(j-1)])
+        i = j
+      end
+      # end while
+      if _end and i < n
+        handle_data(@rawdata[i..(n-1)])
+        i = n
+      end
+      @rawdata = rawdata[i..-1]
+    end
+
+    def parse_comment(i)
+      rawdata = @rawdata
+      if rawdata[i, 4] != '<!--'
+        raise RuntimeError, 'unexpected call to handle_comment'
+      end
+      match = rawdata.index(Commentclose, i)
+      return nil unless match
+      matched_length = $&.length
+      j = match
+      handle_comment(rawdata[i+4..(j-1)])
+      j = match + matched_length
+      return j-i
+    end
+
+    def parse_starttag(i)
+      rawdata = @rawdata
+      j = rawdata.index(Endbracket, i + 1)
+      return nil unless j
+      attrs = []
+      if rawdata[i+1] == ?> #
+        # SGML shorthand: <> == <last open tag seen>
+        k = j
+        tag = @lasttag
+      else
+        match = rawdata.index(Tagfind, i + 1)
+        unless match
+          raise RuntimeError, 'unexpected call to parse_starttag'
+        end
+        k = i + 1 + ($&.length)
+        tag = $&.downcase
+        @lasttag = tag
+      end
+      while k < j
+        break unless rawdata.index(Attrfind, k)
+        matched_length = $&.length
+        attrname, rest, attrvalue = $1, $2, $3
+        if not rest
+          attrvalue = '' # was: = attrname
+        elsif (attrvalue[0] == ?' && attrvalue[-1] == ?') or
+            (attrvalue[0] == ?" && attrvalue[-1,1] == ?")
+          attrvalue = attrvalue[1..-2]
+        end
+        attrs << [attrname.downcase, attrvalue]
+        k += matched_length
+      end
+      if rawdata[j] == ?> #
+        j += 1
+      end
+      finish_starttag(tag, attrs)
+      return j
+    end
+
+    def parse_endtag(i)
+      rawdata = @rawdata
+      j = rawdata.index(Endbracket, i + 1)
+      return nil unless j
+      tag = (rawdata[i+2..j-1].strip).downcase
+      if rawdata[j] == ?> #
+        j += 1
+      end
+      finish_endtag(tag)
+      return j
+    end
+
+    def finish_starttag(tag, attrs)
+      method = 'start_' + tag
+      if self.respond_to?(method)
+        @stack << tag
+        handle_starttag(tag, method, attrs)
+        return 1
+      else
+        method = 'do_' + tag
+        if self.respond_to?(method)
+          handle_starttag(tag, method, attrs)
+          return 0
+        else
+          unknown_starttag(tag, attrs)
+          return -1
+        end
+      end
+    end
+
+    def finish_endtag(tag)
+      if tag == ''
+        found = @stack.length - 1
+        if found < 0
+          unknown_endtag(tag)
+          return
+        end
+      else
+        unless @stack.include? tag
+          method = 'end_' + tag
+          unless self.respond_to?(method)
+            unknown_endtag(tag)
+          end
+          return
+        end
+        found = @stack.index(tag) #or @stack.length
+      end
+      while @stack.length > found
+        tag = @stack[-1]
+        method = 'end_' + tag
+        if respond_to?(method)
+          handle_endtag(tag, method)
+        else
+          unknown_endtag(tag)
+        end
+        @stack.pop
+      end
+    end
+
+    def parse_special(i)
+      rawdata = @rawdata
+      match = rawdata.index(Endbracket, i+1)
+      return nil unless match
+      matched_length = $&.length
+      handle_special(rawdata[i+1..(match-1)])
+      return match - i + matched_length
+    end
+
+    def handle_starttag(tag, method, attrs)
+      self.send(method, attrs)
+    end
+
+    def handle_endtag(tag, method)
+      self.send(method)
+    end
+
+    def report_unbalanced(tag)
+      if @verbose
+        print '*** Unbalanced </' + tag + '>', "\n"
+        print '*** Stack:', self.stack, "\n"
+      end
+    end
+
+    def handle_charref(name)
+      n = name.to_i
+      if !(0 <= n && n <= 255)
+        unknown_charref(name)
+        return
+      end
+      handle_data(n.chr)
+    end
+
+    def handle_entityref(name)
+      table = Entitydefs
+      if table.include?(name)
+        handle_data(table[name])
+      else
+        unknown_entityref(name)
+        return
+      end
+    end
+
+    def handle_data(data)
+    end
+
+    def handle_comment(data)
+    end
+
+    def handle_special(data)
+    end
+
+    def unknown_starttag(tag, attrs)
+    end
+    def unknown_endtag(tag)
+    end
+    def unknown_charref(ref)
+    end
+    def unknown_entityref(ref)
+    end
+  end
+end

data/lib/feedparser/text-output.rb

@@ -0,0 +1,108 @@
+require 'feedparser'
+require 'feedparser/html2text-parser'
+require 'feedparser/filesizes'
+
+class String
+  # Convert an HTML text to plain text
+  def html2text(wrapto = false)
+    text = self.clone
+    # parse HTML
+    p = FeedParser::HTML2TextParser::new(true)
+    p.feed(text)
+    p.close
+    text = p.savedata
+    # remove leading and trailing whilespace
+    text.gsub!(/\A\s*/m, '')
+    text.gsub!(/\s*\Z/m, '')
+    # remove whitespace around \n
+    text.gsub!(/ *\n/m, "\n")
+    text.gsub!(/\n */m, "\n")
+    # and duplicates \n
+    text.gsub!(/\n\n+/m, "\n\n")
+    # and remove duplicated whitespace
+    text.gsub!(/[ \t]+/, ' ')
+
+    # finally, wrap the text if requested
+    return wrap_text(text, wrapto) if wrapto
+    text
+  end
+
+  def wrap_text(text, wrapto = 72)
+    text.gsub(/(.{1,#{wrapto}})( +|$)\n?/, "\\1\\2\n")
+  end
+end
+
+module FeedParser
+  class Feed
+    def to_text(localtime = true, wrapto = false)
+      s = ''
+      s += "Type: #{@type}\n"
+      s += "Encoding: #{@encoding}\n"
+      s += "Title: #{@title}\n"
+      s += "Link: #{@link}\n"
+      if @description
+        s += "Description: #{@description.html2text}\n"
+      else
+        s += "Description:\n"
+      end
+      s += "Creator: #{@creator}\n"
+      s += "\n"
+      @items.each do |i|
+        s += '*' * 40 + "\n"
+        s += i.to_text(localtime, wrapto)
+      end
+      s
+    end
+  end
+
+  class FeedItem
+    def to_text(localtime = true, wrapto = false, header = true)
+      s = ""
+      if header
+        s += "Item: "
+        s += @title if @title
+        s += "\n<#{@link}>" if @link
+        if @date
+          if localtime
+            s += "\nDate: #{@date.to_s}"
+          else
+            s += "\nDate: #{@date.getutc.to_s}"
+          end
+        end
+        s += "\n"
+      else
+        s += "<#{@link}>\n\n" if @link
+      end
+      s += "#{@content.html2text(wrapto).chomp}\n" if @content
+      if @enclosures and @enclosures.length > 0
+        s += "\nFiles:"
+        @enclosures.each do |e|
+          s += "\n #{e[0]} (#{e[1].to_i.to_human_readable}, #{e[2]})"
+        end
+      end
+      if not header
+        s += "-- "
+      end
+      s += "\nFeed: "
+      s += @feed.title if @feed.title
+      s += "\n<#{@feed.link}>" if @feed.link
+      if not header
+        s += "\nItem: "
+        s += @title if @title
+        s += "\n<#{@link}>" if @link
+        if @date
+          if localtime
+            s += "\nDate: #{@date.to_s}"
+          else
+            s += "\nDate: #{@date.getutc.to_s}"
+          end
+        end
+      end
+      s += "\nAuthor: #{creator}" if creator
+      s += "\nSubject: #{@subject}" if @subject
+      s += "\nFiled under: #{@categories.join(', ')}" unless @categories.empty?
+      s += "\n" # final newline, for compat with history
+      s 
+    end
+  end
+end