feedtools 0.1.0 → 0.2.0

data/lib/feed_tools/vendor/htree/raw_string.rb

@@ -0,0 +1,124 @@
+# :stopdoc:
+require 'htree/modules'
+require 'htree/fstr'
+
+module HTree # :nodoc:
+  module Node # :nodoc:
+    # raw_string returns a source string recorded by parsing.
+    # It returns +nil+ if the node is constructed not via parsing.
+    def raw_string
+      catch(:raw_string_tag) {
+        return raw_string_internal('')
+      }
+      nil
+    end
+  end
+
+  class Doc # :nodoc:
+    def raw_string_internal(result)
+      @children.each {|n|
+        n.raw_string_internal(result)
+      }
+    end
+  end
+
+  class Elem # :nodoc:
+    def raw_string_internal(result)
+      @stag.raw_string_internal(result)
+      @children.each {|n| n.raw_string_internal(result) }
+      @etag.raw_string_internal(result) if @etag
+    end
+  end
+
+  module Tag # :nodoc:
+    def init_raw_string() @raw_string = nil end
+    def raw_string=(arg) @raw_string = HTree.frozen_string(arg) end
+    def raw_string_internal(result)
+      throw :raw_string_tag if !@raw_string
+      result << @raw_string
+    end
+  end
+
+  module Leaf # :nodoc:
+    def init_raw_string() @raw_string = nil end
+    def raw_string=(arg) @raw_string = HTree.frozen_string(arg) end
+    def raw_string_internal(result)
+      throw :raw_string_tag if !@raw_string
+      result << @raw_string
+    end
+  end
+
+  class Text # :nodoc:
+    def raw_string=(arg)
+      if arg == @rcdata then
+        @raw_string = @rcdata
+      else
+        super
+      end
+    end
+  end
+
+  module Node # :nodoc:
+    def eliminate_raw_string
+      raise NotImplementedError
+    end
+  end
+
+  class Doc # :nodoc:
+    def eliminate_raw_string
+      Doc.new(@children.map {|c| c.eliminate_raw_string })
+    end
+  end
+
+  class Elem # :nodoc:
+    def eliminate_raw_string
+      Elem.new!(
+        @stag.eliminate_raw_string,
+        @empty ? nil : @children.map {|c| c.eliminate_raw_string },
+        @etag && @etag.eliminate_raw_string)
+    end
+  end
+
+  class Text # :nodoc:
+    def eliminate_raw_string
+      Text.new_internal(@rcdata)
+    end
+  end
+
+  class STag # :nodoc:
+    def eliminate_raw_string
+      STag.new(@qualified_name, @attributes, @inherited_context)
+    end
+  end
+
+  class ETag # :nodoc:
+    def eliminate_raw_string
+      self.class.new(@qualified_name)
+    end
+  end
+
+  class XMLDecl # :nodoc:
+    def eliminate_raw_string
+      XMLDecl.new(@version, @encoding, @standalone)
+    end
+  end
+
+  class DocType # :nodoc:
+    def eliminate_raw_string
+      DocType.new(@root_element_name, @public_identifier, @system_identifier)
+    end
+  end
+
+  class ProcIns # :nodoc:
+    def eliminate_raw_string
+      ProcIns.new(@target, @content)
+    end
+  end
+
+  class Comment # :nodoc:
+    def eliminate_raw_string
+      Comment.new(@content)
+    end
+  end
+end
+# :startdoc:

data/lib/feed_tools/vendor/htree/regexp-util.rb

@@ -0,0 +1,15 @@
+# :stopdoc:
+class Regexp # :nodoc:
+  def disable_capture
+    re = ''
+    self.source.scan(/\\.|[^\\\(]+|\(\?|\(/m) {|s|
+      if s == '('
+        re << '(?:'
+      else
+        re << s
+      end
+    }
+    Regexp.new(re, self.options, self.kcode)
+  end
+end
+# :startdoc:

data/lib/feed_tools/vendor/htree/rexml.rb

@@ -0,0 +1,130 @@
+# = REXML Tree Generator
+#
+# HTree::Node#to_rexml is used for converting HTree to REXML.
+#
+# == Method Summary
+#
+# - HTree::Node#to_rexml -> REXML::Child
+#
+# == Example
+#
+#   HTree.parse(...).to_rexml #=> REXML::Document
+#
+# == Comparison between HTree and REXML.
+#
+# - HTree parser is permissive HTML/XML parser.
+#   REXML parser is strict XML parser.
+#   HTree is recommended if you need to parse realworld HTML.
+#   REXML is recommended if you need strict error checking.
+# - HTree object is immutable.
+#   REXML object is mutable.
+#   REXML should be used if you need modification.
+#
+
+# :stopdoc:
+require 'htree/modules'
+require 'htree/output' # HTree::DocType#generate_content
+
+module HTree # :nodoc:
+  module Node # :nodoc:
+    # convert to REXML tree.
+    def to_rexml
+      require 'rexml/document'
+      to_rexml_internal(nil, DefaultContext)
+    end
+  end
+
+  class Doc # :nodoc:
+    def to_rexml_internal(parent, context)
+      raise ArgumentError, "parent must be nil" if parent != nil
+      result = REXML::Document.new
+      self.children.each {|c|
+        c.to_rexml_internal(result, context)
+      }
+      result
+    end
+  end
+
+  class Elem # :nodoc:
+    def to_rexml_internal(parent, context)
+      ename = self.element_name
+      ns_decl = {}
+      if context.namespace_uri(ename.namespace_prefix) != ename.namespace_uri
+        ns_decl[ename.namespace_prefix] = ename.namespace_uri
+      end
+
+      if ename.namespace_prefix
+        result = REXML::Element.new("#{ename.namespace_prefix}:#{ename.local_name}", parent)
+      else
+        result = REXML::Element.new(ename.local_name, parent)
+      end
+
+      self.each_attribute {|aname, atext|
+        if aname.namespace_prefix
+          if context.namespace_uri(aname.namespace_prefix) != aname.namespace_uri
+            ns_decl[aname.namespace_prefix] = aname.namespace_uri
+          end
+          result.add_attribute("#{aname.namespace_prefix}:#{aname.local_name}", atext.to_s)
+        else
+          result.add_attribute(aname.local_name, atext.to_s)
+        end
+      }
+
+      ns_decl.each {|k, v|
+        if k
+          result.add_namespace(k, v)
+        else
+          result.add_namespace(v)
+        end
+      }
+      context = context.subst_namespaces(ns_decl)
+
+      self.children.each {|c|
+        c.to_rexml_internal(result, context)
+      }
+      result
+    end
+  end
+
+  class Text # :nodoc:
+    def to_rexml_internal(parent, context)
+      rcdata = self.rcdata.gsub(/[<>]/) { Encoder::ChRef[$&] }
+      REXML::Text.new(rcdata, true, parent, true)
+    end
+  end
+
+  class XMLDecl # :nodoc:
+    def to_rexml_internal(parent, context)
+      r = REXML::XMLDecl.new(self.version, self.encoding, self.standalone)
+      parent << r if parent
+      r
+    end
+  end
+
+  class DocType # :nodoc:
+    def to_rexml_internal(parent, context)
+      REXML::DocType.new([self.root_element_name, self.generate_content], parent)
+    end
+  end
+
+  class ProcIns # :nodoc:
+    def to_rexml_internal(parent, context)
+      r = REXML::Instruction.new(self.target, self.content)
+      parent << r if parent
+      r
+    end
+  end
+
+  class Comment # :nodoc:
+    def to_rexml_internal(parent, context)
+      REXML::Comment.new(self.content, parent)
+    end
+  end
+
+  class BogusETag # :nodoc:
+    def to_rexml_internal(parent, context)
+      nil
+    end
+  end
+end
+# :startdoc:

data/lib/feed_tools/vendor/htree/scan.rb

@@ -0,0 +1,166 @@
+# :stopdoc:
+require 'htree/htmlinfo'
+require 'htree/regexp-util'
+require 'htree/fstr'
+
+module HTree # :nodoc:
+  module Pat # :nodoc:
+    NameChar = /[-A-Za-z0-9._:]/
+    Name = /[A-Za-z_:]#{NameChar}*/
+    Nmtoken = /#{NameChar}+/
+
+    Comment_C = /<!--(.*?)-->/m
+    Comment = Comment_C.disable_capture
+    CDATA_C = /<!\[CDATA\[(.*?)\]\]>/m
+    CDATA = CDATA_C.disable_capture
+
+    QuotedAttr_C = /(#{Name})\s*=\s*(?:"([^"]*)"|'([^']*)')/
+    QuotedAttr = QuotedAttr_C.disable_capture
+    ValidAttr_C = /(#{Name})\s*=\s*(?:"([^"]*)"|'([^']*)'|(#{NameChar}*))|(#{Nmtoken})/
+    ValidAttr = ValidAttr_C.disable_capture
+    InvalidAttr1_C = /(#{Name})\s*=\s*(?:'([^'<>]*)'|"([^"<>]*)"|([^\s<>"']*))|(#{Nmtoken})/
+    InvalidAttr1 = InvalidAttr1_C.disable_capture
+    InvalidAttr1End_C =   /(#{Name})(?:\s*=\s*(?:'([^'<>]*)|"([^"<>]*)))/
+    InvalidAttr1End = InvalidAttr1End_C.disable_capture
+
+    QuotedStartTag_C = /<(#{Name})((?:\s+#{QuotedAttr})*)\s*>/
+    QuotedStartTag = QuotedStartTag_C.disable_capture
+    ValidStartTag_C = /<(#{Name})((?:\s+#{ValidAttr})*)\s*>/
+    ValidStartTag = ValidStartTag_C.disable_capture
+    InvalidStartTag_C = /<(#{Name})((?:(?:\b|\s+)#{InvalidAttr1})*)((?:\b|\s+)#{InvalidAttr1End})?\s*>/
+    InvalidStartTag = InvalidStartTag_C.disable_capture
+    StartTag = /#{QuotedStartTag}|#{ValidStartTag}|#{InvalidStartTag}/
+
+    QuotedEmptyTag_C = %r{<(#{Name})((?:\s+#{QuotedAttr})*)\s*/>}
+    QuotedEmptyTag = QuotedEmptyTag_C.disable_capture
+    ValidEmptyTag_C = %r{<(#{Name})((?:\s+#{ValidAttr})*)\s*/>}
+    ValidEmptyTag = ValidEmptyTag_C.disable_capture
+    InvalidEmptyTag_C = %r{<(#{Name})((?:(?:\b|\s+)#{InvalidAttr1})*)((?:\b|\s+)#{InvalidAttr1End})?\s*/>}
+    InvalidEmptyTag = InvalidEmptyTag_C.disable_capture
+    EmptyTag = /#{QuotedEmptyTag}|#{ValidEmptyTag}|#{InvalidEmptyTag}/
+
+    EndTag_C = %r{</(#{Name})\s*>}
+    EndTag = EndTag_C.disable_capture
+
+    XmlVersionNum = /[a-zA-Z0-9_.:-]+/
+    XmlVersionInfo_C = /\s+version\s*=\s*(?:'(#{XmlVersionNum})'|"(#{XmlVersionNum})")/
+    XmlVersionInfo = XmlVersionInfo_C.disable_capture
+    XmlEncName = /[A-Za-z][A-Za-z0-9._-]*/
+    XmlEncodingDecl_C = /\s+encoding\s*=\s*(?:"(#{XmlEncName})"|'(#{XmlEncName})')/
+    XmlEncodingDecl = XmlEncodingDecl_C.disable_capture
+    XmlSDDecl_C = /\s+standalone\s*=\s*(?:'(yes|no)'|"(yes|no)")/
+    XmlSDDecl = XmlSDDecl_C.disable_capture
+    XmlDecl_C = /<\?xml#{XmlVersionInfo_C}#{XmlEncodingDecl_C}?#{XmlSDDecl_C}?\s*\?>/
+    XmlDecl = /<\?xml#{XmlVersionInfo}#{XmlEncodingDecl}?#{XmlSDDecl}?\s*\?>/
+
+    # xxx: internal DTD subset is not recognized: '[' (markupdecl | DeclSep)* ']' S?)?
+    SystemLiteral_C = /"([^"]*)"|'([^']*)'/
+    PubidLiteral_C = %r{"([\sa-zA-Z0-9\-'()+,./:=?;!*\#@$_%]*)"|'([\sa-zA-Z0-9\-()+,./:=?;!*\#@$_%]*)'}
+    ExternalID_C = /(?:SYSTEM|PUBLIC\s+#{PubidLiteral_C})(?:\s+#{SystemLiteral_C})?/
+    DocType_C = /<!DOCTYPE\s+(#{Name})(?:\s+#{ExternalID_C})?\s*(?:\[.*?\]\s*)?>/m
+    DocType = DocType_C.disable_capture
+
+    XmlProcIns_C = /<\?(#{Name})(?:\s+(.*?))?\?>/m
+    XmlProcIns = XmlProcIns_C.disable_capture
+    #ProcIns = /<\?([^>]*)>/m
+  end
+
+  def HTree.scan(input, is_xml=false)
+    is_html = false
+    cdata_content = nil
+    text_start = 0
+    first_element = true
+    index_xmldecl = 1
+    index_doctype = 2
+    index_xmlprocins = 3
+    index_quotedstarttag = 4
+    index_quotedemptytag = 5
+    index_starttag = 6
+    index_endtag = 7
+    index_emptytag = 8
+    index_comment = 9
+    index_cdata = 10
+    input.scan(/(#{Pat::XmlDecl})
+               |(#{Pat::DocType})
+               |(#{Pat::XmlProcIns})
+               |(#{Pat::QuotedStartTag})
+               |(#{Pat::QuotedEmptyTag})
+               |(#{Pat::StartTag})
+               |(#{Pat::EndTag})
+               |(#{Pat::EmptyTag})
+               |(#{Pat::Comment})
+               |(#{Pat::CDATA})
+               /ox) {
+      match = $~
+      if cdata_content
+        str = $&
+        if match.begin(index_endtag) && str[Pat::Name] == cdata_content
+          text_end = match.begin(0)
+          if text_start < text_end
+            yield [:text_cdata_content, HTree.frozen_string(input[text_start...text_end])]
+          end
+          yield [:etag, HTree.frozen_string(str)]
+          text_start = match.end(0)
+          cdata_content = nil
+        end
+      else
+        str = match[0]
+        text_end = match.begin(0)
+        if text_start < text_end
+          yield [:text_pcdata, HTree.frozen_string(input[text_start...text_end])]
+        end
+        text_start = match.end(0)
+        if match.begin(index_xmldecl)
+          yield [:xmldecl, HTree.frozen_string(str)]
+          is_xml = true
+        elsif match.begin(index_doctype)
+          Pat::DocType_C =~ str
+          root_element_name = $1
+          public_identifier = $2 || $3
+          system_identifier = $4 || $5
+          is_html = true if /\Ahtml\z/i =~ root_element_name
+          is_xml = true if public_identifier && %r{\A-//W3C//DTD XHTML } =~ public_identifier
+          yield [:doctype, HTree.frozen_string(str)]
+        elsif match.begin(index_xmlprocins)
+          yield [:procins, HTree.frozen_string(str)]
+        elsif match.begin(index_starttag) || match.begin(index_quotedstarttag)
+          yield stag = [:stag, HTree.frozen_string(str)]
+          tagname = str[Pat::Name]
+          if first_element
+            if /\A(?:html|head|title|isindex|base|script|style|meta|link|object)\z/i =~ tagname
+              is_html = true
+            else
+              is_xml = true
+            end
+            first_element = false
+          end
+          if !is_xml && ElementContent[tagname] == :CDATA
+            cdata_content = tagname
+          end
+        elsif match.begin(index_endtag)
+          yield [:etag, HTree.frozen_string(str)]
+        elsif match.begin(index_emptytag) || match.begin(index_quotedemptytag)
+          yield [:emptytag, HTree.frozen_string(str)]
+          first_element = false
+          #is_xml = true
+        elsif match.begin(index_comment)
+          yield [:comment, HTree.frozen_string(str)]
+        elsif match.begin(index_cdata)
+          yield [:text_cdata_section, HTree.frozen_string(str)]
+        else
+          raise Exception, "unknown match [bug]"
+        end
+      end
+    }
+    text_end = input.length
+    if text_start < text_end
+      if cdata_content
+        yield [:text_cdata_content, HTree.frozen_string(input[text_start...text_end])]
+      else
+        yield [:text_pcdata, HTree.frozen_string(input[text_start...text_end])]
+      end
+    end
+    return is_xml, is_html
+  end
+end
+# :startdoc:

data/lib/feed_tools/vendor/htree/tag.rb

@@ -0,0 +1,111 @@
+# :stopdoc:
+require 'htree/raw_string'
+require 'htree/text'
+require 'htree/scan' # for Pat::Name and Pat::Nmtoken
+require 'htree/context'
+require 'htree/name'
+require 'htree/fstr'
+
+module HTree # :nodoc:
+  class STag # :nodoc:
+    def initialize(name, attributes=[], inherited_context=DefaultContext)
+      init_raw_string
+      # normalize xml declaration name and attribute value.
+      attributes = attributes.map {|aname, val|
+        if !(Name === aname) && /\A(?:#{Pat::Name}?\{.*\})?#{Pat::Nmtoken}\z/o !~ aname
+          raise HTree::Error, "invalid attribute name: #{aname.inspect}"
+        end
+        if !(Name === aname) && /\Axmlns(?:\z|:)/ =~ aname
+          aname = Name.parse_attribute_name(aname, nil)
+        end
+        val = val.to_node if HTree::Location === val
+        val = Text.new(val) unless Text === val
+        [aname, val]
+      }
+
+      @inherited_context = inherited_context
+      @xmlns_decls = {}
+
+      # validate namespace consistency of given Name objects.
+      if Name === name
+        @xmlns_decls[name.namespace_prefix] = name.namespace_uri
+      end
+      attributes.each {|aname, text|
+        next unless Name === aname
+        next if aname.xmlns?
+        if aname.namespace_prefix && aname.namespace_uri
+          if @xmlns_decls.include? aname.namespace_prefix
+            if @xmlns_decls[aname.namespace_prefix] != aname.namespace_uri
+              raise ArgumentError, "inconsistent namespace use: #{aname.namespace_prefix} is used as #{@xmlns_decls[aname.namespace_prefix]} and #{aname.namespace_uri}"
+            end
+          else
+            @xmlns_decls[aname.namespace_prefix] = aname.namespace_uri
+          end
+        end
+      }
+
+      attributes.each {|aname, text|
+        next unless Name === aname
+        next unless aname.xmlns?
+        next if @xmlns_decls.include? aname.local_name
+        if aname.local_name
+          @xmlns_decls[aname.local_name] = text.to_s
+        else
+          uri = text.to_s
+          @xmlns_decls[nil] = uri
+        end
+      }
+
+      @context = make_context(@inherited_context)
+
+      if Name === name
+        @name = name
+      else
+        @name = Name.parse_element_name(name, @context)
+      end
+
+      @attributes = attributes.map {|aname, text|
+        aname = Name.parse_attribute_name(aname, @context) unless Name === aname
+        if !aname.namespace_prefix && !aname.namespace_uri.empty?
+          # xxx: should recover error?
+          raise HTree::Error, "global attribute without namespace prefix: #{aname.inspect}"
+        end
+        [aname, text]
+      }
+      @attributes.freeze
+    end
+    attr_reader :attributes, :inherited_context, :context
+
+    def element_name
+      @name
+    end
+
+    def make_context(inherited_context)
+      inherited_context.subst_namespaces(@xmlns_decls)
+    end
+
+    def each_namespace_attribute
+      @xmlns_decls.each {|name, uri|
+        yield name, uri
+      }
+      nil
+    end
+
+    def each_attribute
+      @attributes.each {|name, text|
+        next if name.xmlns?
+        yield name, text
+      }
+      nil
+    end
+  end
+
+  class ETag # :nodoc:
+    def initialize(qualified_name)
+      init_raw_string
+      @qualified_name = HTree.frozen_string(qualified_name)
+    end
+    attr_reader :qualified_name
+  end
+end
+# :startdoc: