htree 0.7.0

data/lib/htree/raw_string.rb

@@ -0,0 +1,127 @@
+require 'htree/modules'
+require 'htree/fstr'
+
+module HTree
+  module Node
+    # raw_string returns a source string recorded by parsing.
+    # It returns +nil+ if the node is constructed not via parsing.
+    def raw_string
+      catch(:raw_string_tag) {
+        return raw_string_internal('')
+      }
+      nil
+    end
+  end
+
+  # :stopdoc:
+  class Doc
+    def raw_string_internal(result)
+      @children.each {|n|
+        n.raw_string_internal(result)
+      }
+    end
+  end
+
+  class Elem
+    def raw_string_internal(result)
+      @stag.raw_string_internal(result)
+      @children.each {|n| n.raw_string_internal(result) }
+      @etag.raw_string_internal(result) if @etag
+    end
+  end
+
+  module Tag
+    def init_raw_string() @raw_string = nil end
+    def raw_string=(arg) @raw_string = HTree.frozen_string(arg) end
+    def raw_string_internal(result)
+      throw :raw_string_tag if !@raw_string
+      result << @raw_string
+    end
+  end
+
+  module Leaf
+    def init_raw_string() @raw_string = nil end
+    def raw_string=(arg) @raw_string = HTree.frozen_string(arg) end
+    def raw_string_internal(result)
+      throw :raw_string_tag if !@raw_string
+      result << @raw_string
+    end
+  end
+
+  class Text
+    def raw_string=(arg)
+      if arg == @rcdata then
+        @raw_string = @rcdata
+      else
+        super
+      end
+    end
+  end
+  # :startdoc:
+
+  module Node
+    def eliminate_raw_string
+      raise NotImplementedError
+    end
+  end
+
+  # :stopdoc:
+
+  class Doc
+    def eliminate_raw_string
+      Doc.new(@children.map {|c| c.eliminate_raw_string })
+    end
+  end
+
+  class Elem
+    def eliminate_raw_string
+      Elem.new!(
+        @stag.eliminate_raw_string,
+        @empty ? nil : @children.map {|c| c.eliminate_raw_string },
+        @etag && @etag.eliminate_raw_string)
+    end
+  end
+
+  class Text
+    def eliminate_raw_string
+      Text.new_internal(@rcdata)
+    end
+  end
+
+  class STag
+    def eliminate_raw_string
+      STag.new(@qualified_name, @attributes, @inherited_context)
+    end
+  end
+
+  class ETag
+    def eliminate_raw_string
+      self.class.new(@qualified_name)
+    end
+  end
+
+  class XMLDecl
+    def eliminate_raw_string
+      XMLDecl.new(@version, @encoding, @standalone)
+    end
+  end
+
+  class DocType
+    def eliminate_raw_string
+      DocType.new(@root_element_name, @public_identifier, @system_identifier)
+    end
+  end
+
+  class ProcIns
+    def eliminate_raw_string
+      ProcIns.new(@target, @content)
+    end
+  end
+
+  class Comment
+    def eliminate_raw_string
+      Comment.new(@content)
+    end
+  end
+  # :startdoc:
+end

data/lib/htree/regexp-util.rb

@@ -0,0 +1,19 @@
+class Regexp
+  def disable_capture
+    re = ''
+    self.source.scan(/\\.|[^\\\(]+|\(\?|\(/m) {|s|
+      if s == '('
+        re << '(?:'
+      else
+        re << s
+      end
+    }
+    if re.respond_to? :force_encoding
+      re.force_encoding(self.encoding)
+      Regexp.new(re, self.options)
+    else
+      Regexp.new(re, self.options, self.kcode)
+    end
+  end
+end
+

data/lib/htree/rexml.rb

@@ -0,0 +1,131 @@
+# = REXML Tree Generator
+#
+# HTree::Node#to_rexml is used for converting HTree to REXML.
+#
+# == Method Summary
+#
+# - HTree::Node#to_rexml -> REXML::Child
+#
+# == Example
+#
+#   HTree.parse(...).to_rexml #=> REXML::Document
+#
+# == Comparison between HTree and REXML.
+#
+# - HTree parser is permissive HTML/XML parser.
+#   REXML parser is strict XML parser.
+#   HTree is recommended if you need to parse realworld HTML.
+#   REXML is recommended if you need strict error checking.
+# - HTree object is immutable.
+#   REXML object is mutable.
+#   REXML should be used if you need modification.
+#
+require 'htree/modules'
+require 'htree/output' # HTree::DocType#generate_content
+
+module HTree
+  module Node
+    # convert to REXML tree.
+    def to_rexml
+      require 'rexml/document'
+      to_rexml_internal(nil, DefaultContext)
+    end
+  end
+
+  # :stopdoc:
+
+  class Doc
+    def to_rexml_internal(parent, context)
+      raise ArgumentError, "parent must be nil" if parent != nil
+      result = REXML::Document.new
+      self.children.each {|c|
+        c.to_rexml_internal(result, context)
+      }
+      result
+    end
+  end
+
+  class Elem
+    def to_rexml_internal(parent, context)
+      ename = self.element_name
+      ns_decl = {}
+      if context.namespace_uri(ename.namespace_prefix) != ename.namespace_uri
+        ns_decl[ename.namespace_prefix] = ename.namespace_uri
+      end
+
+      if ename.namespace_prefix
+        result = REXML::Element.new("#{ename.namespace_prefix}:#{ename.local_name}", parent)
+      else
+        result = REXML::Element.new(ename.local_name, parent)
+      end
+
+      self.each_attribute {|aname, atext|
+        if aname.namespace_prefix
+          if context.namespace_uri(aname.namespace_prefix) != aname.namespace_uri
+            ns_decl[aname.namespace_prefix] = aname.namespace_uri
+          end
+          result.add_attribute("#{aname.namespace_prefix}:#{aname.local_name}", atext.to_s)
+        else
+          result.add_attribute(aname.local_name, atext.to_s)
+        end
+      }
+
+      ns_decl.each {|k, v|
+        if k
+          result.add_namespace(k, v)
+        else
+          result.add_namespace(v)
+        end
+      }
+      context = context.subst_namespaces(ns_decl)
+
+      self.children.each {|c|
+        c.to_rexml_internal(result, context)
+      }
+      result
+    end
+  end
+
+  class Text
+    def to_rexml_internal(parent, context)
+      rcdata = self.rcdata.gsub(/[<>]/) { Encoder::ChRef[$&] }
+      REXML::Text.new(rcdata, true, parent, true)
+    end
+  end
+
+  class XMLDecl
+    def to_rexml_internal(parent, context)
+      r = REXML::XMLDecl.new(self.version, self.encoding, self.standalone)
+      parent << r if parent
+      r
+    end
+  end
+
+  class DocType
+    def to_rexml_internal(parent, context)
+      REXML::DocType.new([self.root_element_name, self.generate_content], parent)
+    end
+  end
+
+  class ProcIns
+    def to_rexml_internal(parent, context)
+      r = REXML::Instruction.new(self.target, self.content)
+      parent << r if parent
+      r
+    end
+  end
+
+  class Comment
+    def to_rexml_internal(parent, context)
+      REXML::Comment.new(self.content, parent)
+    end
+  end
+
+  class BogusETag
+    def to_rexml_internal(parent, context)
+      nil
+    end
+  end
+
+  # :startdoc:
+end

data/lib/htree/scan.rb

@@ -0,0 +1,176 @@
+require 'htree/htmlinfo'
+require 'htree/regexp-util'
+require 'htree/fstr'
+
+module HTree
+  # :stopdoc:
+  module Pat
+    NameChar = /[-A-Za-z0-9._:]/
+    Name = /[A-Za-z_:]#{NameChar}*/
+    Nmtoken = /#{NameChar}+/
+
+    Comment_C = /<!--(.*?)-->/m
+    Comment = Comment_C.disable_capture
+    CDATA_C = /<!\[CDATA\[(.*?)\]\]>/m
+    CDATA = CDATA_C.disable_capture
+
+    QuotedAttr_C = /(#{Name})\s*=\s*(?:"([^"]*)"|'([^']*)')/
+    QuotedAttr = QuotedAttr_C.disable_capture
+    ValidAttr_C = /(#{Name})\s*=\s*(?:"([^"]*)"|'([^']*)'|(#{NameChar}*))|(#{Nmtoken})/
+    ValidAttr = ValidAttr_C.disable_capture
+    InvalidAttr1_C = /(#{Name})\s*=\s*(?:'([^'<>]*)'|"([^"<>]*)"|([^\s<>"']*(?![^\s<>"'])))|(#{Nmtoken})/
+    InvalidAttr1 = InvalidAttr1_C.disable_capture
+    InvalidAttr1End_C =   /(#{Name})(?:\s*=\s*(?:'([^'<>]*)|"([^"<>]*)))/
+    InvalidAttr1End = InvalidAttr1End_C.disable_capture
+
+    QuotedStartTag_C = /<(#{Name})((?:\s+#{QuotedAttr})*)\s*>/
+    QuotedStartTag = QuotedStartTag_C.disable_capture
+    ValidStartTag_C = /<(#{Name})((?:\s+#{ValidAttr})*)\s*>/
+    ValidStartTag = ValidStartTag_C.disable_capture
+    InvalidStartTag_C = /<(#{Name})((?:(?:\b|\s+)#{InvalidAttr1})*)((?:\b|\s+)#{InvalidAttr1End})?\s*>/
+    InvalidStartTag = InvalidStartTag_C.disable_capture
+    StartTag = /#{QuotedStartTag}|#{ValidStartTag}|#{InvalidStartTag}/
+
+    QuotedEmptyTag_C = %r{<(#{Name})((?:\s+#{QuotedAttr})*)\s*/>}
+    QuotedEmptyTag = QuotedEmptyTag_C.disable_capture
+    ValidEmptyTag_C = %r{<(#{Name})((?:\s+#{ValidAttr})*)\s*/>}
+    ValidEmptyTag = ValidEmptyTag_C.disable_capture
+    InvalidEmptyTag_C = %r{<(#{Name})((?:(?:\b|\s+)#{InvalidAttr1})*)((?:\b|\s+)#{InvalidAttr1End})?\s*/>}
+    InvalidEmptyTag = InvalidEmptyTag_C.disable_capture
+    EmptyTag = /#{QuotedEmptyTag}|#{ValidEmptyTag}|#{InvalidEmptyTag}/
+
+    EndTag_C = %r{</(#{Name})\s*>}
+    EndTag = EndTag_C.disable_capture
+
+    XmlVersionNum = /[a-zA-Z0-9_.:-]+/
+    XmlVersionInfo_C = /\s+version\s*=\s*(?:'(#{XmlVersionNum})'|"(#{XmlVersionNum})")/
+    XmlVersionInfo = XmlVersionInfo_C.disable_capture
+    XmlEncName = /[A-Za-z][A-Za-z0-9._-]*/
+    XmlEncodingDecl_C = /\s+encoding\s*=\s*(?:"(#{XmlEncName})"|'(#{XmlEncName})')/
+    XmlEncodingDecl = XmlEncodingDecl_C.disable_capture
+    XmlSDDecl_C = /\s+standalone\s*=\s*(?:'(yes|no)'|"(yes|no)")/
+    XmlSDDecl = XmlSDDecl_C.disable_capture
+    XmlDecl_C = /<\?xml#{XmlVersionInfo_C}#{XmlEncodingDecl_C}?#{XmlSDDecl_C}?\s*\?>/
+    XmlDecl = /<\?xml#{XmlVersionInfo}#{XmlEncodingDecl}?#{XmlSDDecl}?\s*\?>/
+
+    # xxx: internal DTD subset is not recognized: '[' (markupdecl | DeclSep)* ']' S?)?
+    SystemLiteral_C = /"([^"]*)"|'([^']*)'/
+    PubidLiteral_C = %r{"([\sa-zA-Z0-9\-'()+,./:=?;!*\#@$_%]*)"|'([\sa-zA-Z0-9\-()+,./:=?;!*\#@$_%]*)'}
+    ExternalID_C = /(?:SYSTEM|PUBLIC\s+#{PubidLiteral_C})(?:\s+#{SystemLiteral_C})?/
+    DocType_C = /<!DOCTYPE\s+(#{Name})(?:\s+#{ExternalID_C})?\s*(?:\[.*?\]\s*)?>/m
+    DocType = DocType_C.disable_capture
+
+    XmlProcIns_C = /<\?(#{Name})(?:\s+(.*?))?\?>/m
+    XmlProcIns = XmlProcIns_C.disable_capture
+    #ProcIns = /<\?([^>]*)>/m
+  end
+
+  def HTree.scan(input, is_xml=false)
+    is_html = false
+    cdata_content = nil
+    cdata_content_string = nil
+    pcdata = ''
+    first_element = true
+    index_otherstring = 1
+    index_str = 2
+    index_xmldecl = 3
+    index_doctype = 4
+    index_xmlprocins = 5
+    index_quotedstarttag = 6
+    index_quotedemptytag = 7
+    index_starttag = 8
+    index_endtag = 9
+    index_emptytag = 10
+    index_comment = 11
+    index_cdata = 12
+    index_end = 13
+    pat = /\G(.*?)((#{Pat::XmlDecl})
+                  |(#{Pat::DocType})
+                  |(#{Pat::XmlProcIns})
+                  |(#{Pat::QuotedStartTag})
+                  |(#{Pat::QuotedEmptyTag})
+                  |(#{Pat::StartTag})
+                  |(#{Pat::EndTag})
+                  |(#{Pat::EmptyTag})
+                  |(#{Pat::Comment})
+                  |(#{Pat::CDATA})
+                  |(\z))
+          /oxm
+    input.scan(pat) {
+      match = $~
+      if cdata_content
+        cdata_content_string << match[index_otherstring]
+        str = match[index_str]
+        if match[index_endtag] && str[Pat::Name] == cdata_content
+          unless cdata_content_string.empty?
+            yield [:text_cdata_content, HTree.frozen_string(cdata_content_string)]
+          end
+          yield [:etag, HTree.frozen_string(str)]
+          cdata_content = nil
+          cdata_content_string = nil
+        elsif match[index_end]
+          cdata_content_string << str
+          unless cdata_content_string.empty?
+            yield [:text_cdata_content, HTree.frozen_string(cdata_content_string)]
+          end
+          cdata_content = nil
+          cdata_content_string = nil
+        else
+          cdata_content_string << str
+        end
+      else
+        pcdata << match[index_otherstring]
+        str = match[index_str]
+        if !pcdata.empty?
+          yield [:text_pcdata, HTree.frozen_string(pcdata)]
+          pcdata = ''
+        end
+        if match[index_xmldecl]
+          yield [:xmldecl, HTree.frozen_string(str)]
+          is_xml = true
+        elsif match[index_doctype]
+          Pat::DocType_C =~ str
+          root_element_name = $1
+          public_identifier = $2 || $3
+          system_identifier = $4 || $5
+          is_html = true if /\Ahtml\z/i =~ root_element_name
+          is_xml = true if public_identifier && %r{\A-//W3C//DTD XHTML } =~ public_identifier
+          yield [:doctype, HTree.frozen_string(str)]
+        elsif match[index_xmlprocins]
+          yield [:procins, HTree.frozen_string(str)]
+        elsif match[index_starttag] || match[index_quotedstarttag]
+          yield stag = [:stag, HTree.frozen_string(str)]
+          tagname = str[Pat::Name]
+          if first_element
+            if /\A(?:html|head|title|isindex|base|script|style|meta|link|object)\z/i =~ tagname
+              is_html = true
+            else
+              is_xml = true
+            end
+            first_element = false
+          end
+          if !is_xml && ElementContent[tagname] == :CDATA
+            cdata_content = tagname
+            cdata_content_string = ''
+          end
+        elsif match[index_endtag]
+          yield [:etag, HTree.frozen_string(str)]
+        elsif match[index_emptytag] || match[index_quotedemptytag]
+          yield [:emptytag, HTree.frozen_string(str)]
+          first_element = false
+          #is_xml = true
+        elsif match[index_comment]
+          yield [:comment, HTree.frozen_string(str)]
+        elsif match[index_cdata]
+          yield [:text_cdata_section, HTree.frozen_string(str)]
+        elsif match[index_end]
+          # pass
+        else
+          raise Exception, "unknown match [bug]"
+        end
+      end
+    }
+    return is_xml, is_html
+  end
+  # :startdoc:
+end