whistle 0.1

data/vendor/ruby-feedparser-0.5-stripped/lib/feedparser/rexml_patch.rb

@@ -0,0 +1,28 @@
+require 'feedparser/textconverters'
+
+# Patch for REXML
+# Very ugly patch to make REXML error-proof.
+# The problem is REXML uses IConv, which isn't error-proof at all.
+# With those changes, it uses unpack/pack with some error handling
+module REXML
+  module Encoding
+    def decode(str)
+      return str.toUTF8(@encoding)
+    end
+
+    def encode(str)
+      return str
+    end
+
+    def encoding=(enc)
+      return if defined? @encoding and enc == @encoding
+      @encoding = enc || 'utf-8'
+    end
+  end
+
+  class Element
+    def children
+      @children
+    end
+  end
+end

data/vendor/ruby-feedparser-0.5-stripped/lib/feedparser/sgml-parser.rb

@@ -0,0 +1,332 @@
+# A parser for SGML, using the derived class as static DTD.
+# from http://raa.ruby-lang.org/project/html-parser
+module FeedParser
+  class SGMLParser
+    # Regular expressions used for parsing:
+    Interesting = /[&<]/
+    Incomplete = Regexp.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|' +
+                                '<([a-zA-Z][^<>]*|/([a-zA-Z][^<>]*)?|' +
+                                '![^<>]*)?')
+
+    Entityref = /&([a-zA-Z][-.a-zA-Z0-9]*);/
+    Charref = /&#([0-9]+);/
+
+    Starttagopen = /<[>a-zA-Z]/
+    Endtagopen = /<\/[<>a-zA-Z]/
+    Endbracket = /[<>]/
+    Special = /<![^<>]*>/
+    Commentopen = /<!--/
+    Commentclose = /--[ \t\n]*>/
+    Tagfind = /[a-zA-Z][a-zA-Z0-9.-]*/
+    Attrfind = Regexp.compile('[\s,]*([a-zA-Z_][a-zA-Z_0-9.-]*)' +
+                              '(\s*=\s*' +
+                              "('[^']*'" +
+                              '|"[^"]*"' +
+                              '|[-~a-zA-Z0-9,./:+*%?!()_#=]*))?')
+
+    Entitydefs =
+      {'lt'=>'<', 'gt'=>'>', 'amp'=>'&', 'quot'=>'"', 'apos'=>'\''}
+
+    def initialize(verbose=false)
+      @verbose = verbose
+      reset
+    end
+
+    def reset
+      @rawdata = ''
+      @stack = []
+      @lasttag = '???'
+      @nomoretags = false
+      @literal = false
+    end
+
+    def has_context(gi)
+      @stack.include? gi
+    end
+
+    def setnomoretags
+      @nomoretags = true
+      @literal = true
+    end
+
+    def setliteral(*args)
+      @literal = true
+    end
+
+    def feed(data)
+      @rawdata << data
+      goahead(false)
+    end
+
+    def close
+      goahead(true)
+    end
+
+    def goahead(_end)
+      rawdata = @rawdata
+      i = 0
+      n = rawdata.length
+      while i < n
+        if @nomoretags
+          handle_data(rawdata[i..(n-1)])
+          i = n
+          break
+        end
+        j = rawdata.index(Interesting, i)
+        j = n unless j
+        if i < j
+          handle_data(rawdata[i..(j-1)])
+        end
+        i = j
+        break if (i == n)
+        if rawdata[i] == ?< #
+          if rawdata.index(Starttagopen, i) == i
+            if @literal
+              handle_data(rawdata[i, 1])
+              i += 1
+              next
+            end
+            k = parse_starttag(i)
+            break unless k
+            i = k
+            next
+          end
+          if rawdata.index(Endtagopen, i) == i
+            k = parse_endtag(i)
+            break unless k
+            i = k
+            @literal = false
+            next
+          end
+          if rawdata.index(Commentopen, i) == i
+            if @literal
+              handle_data(rawdata[i,1])
+              i += 1
+              next
+            end
+            k = parse_comment(i)
+            break unless k
+            i += k
+            next
+          end
+          if rawdata.index(Special, i) == i
+            if @literal
+              handle_data(rawdata[i, 1])
+              i += 1
+              next
+            end
+            k = parse_special(i)
+            break unless k
+            i += k
+            next
+          end
+        elsif rawdata[i] == ?& #
+          if rawdata.index(Charref, i) == i
+            i += $&.length
+            handle_charref($1)
+            i -= 1 unless rawdata[i-1] == ?;
+            next
+          end
+          if rawdata.index(Entityref, i) == i
+            i += $&.length
+            handle_entityref($1)
+            i -= 1 unless rawdata[i-1] == ?;
+            next
+          end
+        else
+          raise RuntimeError, 'neither < nor & ??'
+        end
+        # We get here only if incomplete matches but
+        # nothing else
+        match = rawdata.index(Incomplete, i)
+        unless match == i
+          handle_data(rawdata[i, 1])
+          i += 1
+          next
+        end
+        j = match + $&.length
+        break if j == n # Really incomplete
+        handle_data(rawdata[i..(j-1)])
+        i = j
+      end
+      # end while
+      if _end and i < n
+        handle_data(@rawdata[i..(n-1)])
+        i = n
+      end
+      @rawdata = rawdata[i..-1]
+    end
+
+    def parse_comment(i)
+      rawdata = @rawdata
+      if rawdata[i, 4] != '<!--'
+        raise RuntimeError, 'unexpected call to handle_comment'
+      end
+      match = rawdata.index(Commentclose, i)
+      return nil unless match
+      matched_length = $&.length
+      j = match
+      handle_comment(rawdata[i+4..(j-1)])
+      j = match + matched_length
+      return j-i
+    end
+
+    def parse_starttag(i)
+      rawdata = @rawdata
+      j = rawdata.index(Endbracket, i + 1)
+      return nil unless j
+      attrs = []
+      if rawdata[i+1] == ?> #
+        # SGML shorthand: <> == <last open tag seen>
+        k = j
+        tag = @lasttag
+      else
+        match = rawdata.index(Tagfind, i + 1)
+        unless match
+          raise RuntimeError, 'unexpected call to parse_starttag'
+        end
+        k = i + 1 + ($&.length)
+        tag = $&.downcase
+        @lasttag = tag
+      end
+      while k < j
+        break unless rawdata.index(Attrfind, k)
+        matched_length = $&.length
+        attrname, rest, attrvalue = $1, $2, $3
+        if not rest
+          attrvalue = '' # was: = attrname
+        elsif (attrvalue[0] == ?' && attrvalue[-1] == ?') or
+            (attrvalue[0] == ?" && attrvalue[-1,1] == ?")
+          attrvalue = attrvalue[1..-2]
+        end
+        attrs << [attrname.downcase, attrvalue]
+        k += matched_length
+      end
+      if rawdata[j] == ?> #
+        j += 1
+      end
+      finish_starttag(tag, attrs)
+      return j
+    end
+
+    def parse_endtag(i)
+      rawdata = @rawdata
+      j = rawdata.index(Endbracket, i + 1)
+      return nil unless j
+      tag = (rawdata[i+2..j-1].strip).downcase
+      if rawdata[j] == ?> #
+        j += 1
+      end
+      finish_endtag(tag)
+      return j
+    end
+
+    def finish_starttag(tag, attrs)
+      method = 'start_' + tag
+      if self.respond_to?(method)
+        @stack << tag
+        handle_starttag(tag, method, attrs)
+        return 1
+      else
+        method = 'do_' + tag
+        if self.respond_to?(method)
+          handle_starttag(tag, method, attrs)
+          return 0
+        else
+          unknown_starttag(tag, attrs)
+          return -1
+        end
+      end
+    end
+
+    def finish_endtag(tag)
+      if tag == ''
+        found = @stack.length - 1
+        if found < 0
+          unknown_endtag(tag)
+          return
+        end
+      else
+        unless @stack.include? tag
+          method = 'end_' + tag
+          unless self.respond_to?(method)
+            unknown_endtag(tag)
+          end
+          return
+        end
+        found = @stack.index(tag) #or @stack.length
+      end
+      while @stack.length > found
+        tag = @stack[-1]
+        method = 'end_' + tag
+        if respond_to?(method)
+          handle_endtag(tag, method)
+        else
+          unknown_endtag(tag)
+        end
+        @stack.pop
+      end
+    end
+
+    def parse_special(i)
+      rawdata = @rawdata
+      match = rawdata.index(Endbracket, i+1)
+      return nil unless match
+      matched_length = $&.length
+      handle_special(rawdata[i+1..(match-1)])
+      return match - i + matched_length
+    end
+
+    def handle_starttag(tag, method, attrs)
+      self.send(method, attrs)
+    end
+
+    def handle_endtag(tag, method)
+      self.send(method)
+    end
+
+    def report_unbalanced(tag)
+      if @verbose
+        print '*** Unbalanced </' + tag + '>', "\n"
+        print '*** Stack:', self.stack, "\n"
+      end
+    end
+
+    def handle_charref(name)
+      n = name.to_i
+      if !(0 <= n && n <= 255)
+        unknown_charref(name)
+        return
+      end
+      handle_data(n.chr)
+    end
+
+    def handle_entityref(name)
+      table = Entitydefs
+      if table.include?(name)
+        handle_data(table[name])
+      else
+        unknown_entityref(name)
+        return
+      end
+    end
+
+    def handle_data(data)
+    end
+
+    def handle_comment(data)
+    end
+
+    def handle_special(data)
+    end
+
+    def unknown_starttag(tag, attrs)
+    end
+    def unknown_endtag(tag)
+    end
+    def unknown_charref(ref)
+    end
+    def unknown_entityref(ref)
+    end
+  end
+end

data/vendor/ruby-feedparser-0.5-stripped/lib/feedparser/text-output.rb

@@ -0,0 +1,83 @@
+require 'feedparser'
+require 'feedparser/html2text-parser'
+require 'feedparser/filesizes'
+
+class String
+  # Convert an HTML text to plain text
+  def html2text
+    text = self.clone
+    # parse HTML
+    p = FeedParser::HTML2TextParser::new(true)
+    p.feed(text)
+    p.close
+    text = p.savedata
+    # remove leading and trailing whilespace
+    text.gsub!(/\A\s*/m, '')
+    text.gsub!(/\s*\Z/m, '')
+    # remove whitespace around \n
+    text.gsub!(/ *\n/m, "\n")
+    text.gsub!(/\n */m, "\n")
+    # and duplicates \n
+    text.gsub!(/\n\n+/m, "\n\n")
+    # and remove duplicated whitespace
+    text.gsub!(/[ \t]+/, ' ')
+    text
+  end
+end
+
+module FeedParser
+  class Feed
+    def to_text(localtime = true)
+      s = ''
+      s += "Type: #{@type}\n"
+      s += "Encoding: #{@encoding}\n"
+      s += "Title: #{@title}\n"
+      s += "Link: #{@link}\n"
+      if @description
+        s += "Description: #{@description.html2text}\n"
+      else
+        s += "Description:\n"
+      end
+      s += "Creator: #{@creator}\n"
+      s += "\n"
+      @items.each do |i|
+        s += '*' * 40 + "\n"
+        s += i.to_text(localtime)
+      end
+      s
+    end
+  end
+
+  class FeedItem
+    def to_text(localtime = true)
+      s = ""
+      s += "Feed: "
+      s += @feed.title + ' ' if @feed.title
+      s += "<#{@feed.link}>" if @feed.link
+      s += "\n"
+      s += "Item: "
+      s += @title + ' ' if @title
+      s += "<#{@link}>" if @link
+      s += "\n"
+      if @date
+        if localtime
+          s += "\nDate: #{@date.to_s}"
+        else
+          s += "\nDate: #{@date.getutc.to_s}"
+        end
+      end
+      s += "\nAuthor: #{@creator}" if @creator
+      s += "\nSubject: #{@subject}" if @subject
+      s += "\nCategory: #{@category}" if @category
+      s += "\n\n"
+      s += "#{@content.html2text}\n" if @content
+      if @enclosures and @enclosures.length > 0
+        s += "Files:\n"
+        @enclosures.each do |e|
+          s += " #{e[0]} (#{e[1].to_i.to_human_readable}, #{e[2]})\n"
+        end
+      end
+      s 
+    end
+  end
+end

data/vendor/ruby-feedparser-0.5-stripped/lib/feedparser/textconverters.rb

@@ -0,0 +1,120 @@
+# for URI::regexp
+require 'uri'
+require 'feedparser/html2text-parser'
+
+# This class provides various converters
+class String
+  # is this text HTML ? search for tags. used by String#text2html
+  def html?
+    return (self =~ /<p>/i) || (self =~ /<\/p>/i) || (self =~ /<br>/i) || (self =~ /<br\s*(\/)?\s*>/i) || (self =~ /<\/a>/i) || (self =~ /<img.*>/i)
+  end
+
+  # returns true if the text contains escaped HTML (with HTML entities). used by String#text2html
+  def escaped_html?
+    return (self =~ /&lt;img src=/i) || (self =~ /&lt;a href=/i) || (self =~ /&lt;br(\/| \/|)&gt;/i) || (self =~ /&lt;p&gt;/i)
+  end
+
+  def escape_html
+    r = self.gsub('&', '&amp;')
+    r = r.gsub('<', '&lt;')
+    r = r.gsub('>', '&gt;')
+    r
+  end
+
+MY_ENTITIES = {}
+FeedParser::HTML2TextParser::entities.each do |k, v|
+  MY_ENTITIES["&#{k};"] = [v].pack('U*')
+  MY_ENTITIES["&##{v};"] = [v].pack('U*')
+end
+
+  # un-escape HTML in the text. used by String#text2html
+  def unescape_html
+    r = self
+    MY_ENTITIES.each do |k, v|
+      r = r.gsub(k, v)
+    end
+    r
+  end
+
+  # convert text to HTML
+  def text2html(feed)
+    text = self.clone
+    realhtml = text.html?
+    eschtml = text.escaped_html?
+    # fix for RSS feeds with both real and escaped html (crazy!):
+    # we take the first one
+    if (realhtml && eschtml)
+      if (realhtml < eschtml)
+        eschtml = nil
+      else
+        realhtml = nil
+      end
+    end
+    if realhtml
+      # do nothing
+    elsif eschtml
+      text = text.unescape_html
+    else
+      # paragraphs
+      text.gsub!(/\A\s*(.*)\Z/m, '<p>\1</p>')
+      text.gsub!(/\s*\n(\s*\n)+\s*/, "</p>\n<p>")
+      # uris
+      text.gsub!(/(#{URI::regexp(['http','ftp','https'])})/,
+          '<a href="\1">\1</a>')
+    end
+    # Handle broken hrefs in <a> and <img>
+    if feed and feed.link
+      text.gsub!(/(\s(src|href)=['"])([^'"]*)(['"])/) do |m|
+        begin
+          first, url, last = $1, $3, $4
+          if (url =~ /^\s*\w+:\/\//) or (url =~ /^\s*\w+:\w/)
+            m
+          elsif url =~ /^\//
+            (first + feed.link.split(/\//)[0..2].join('/') + url + last)
+          else
+            t = feed.link.split(/\//)
+            if t.length == 3 # http://toto with no trailing /
+              (first + feed.link + '/' + url + last)
+            else
+              if feed.link =~ /\/$/
+                (first + feed.link + url + last)
+              else
+                (first + t[0...-1].join('/') + '/' + url + last)
+              end
+            end
+          end
+        rescue
+          m
+        end
+      end
+    end
+    text
+  end
+
+  # Remove white space around the text
+  def rmWhiteSpace!
+    return self.gsub!(/\A\s*/m, '').gsub!(/\s*\Z/m,'')
+  end
+
+  # Convert a text in inputenc to a text in UTF8
+  # must take care of wrong input locales
+  def toUTF8(inputenc)
+    if inputenc.downcase != 'utf-8'
+      # it is said it is not UTF-8. Ensure it is REALLY not UTF-8
+      begin
+        if self.unpack('U*').pack('U*') == self
+          return self
+        end
+      rescue
+        # do nothing
+      end
+      begin
+        return self.unpack('C*').pack('U*')
+      rescue
+        return self #failsafe solution. but a dirty one :-)
+      end
+    else
+      return self
+    end
+  end
+end