html5 0.1.0

data/lib/html5/liberalxmlparser.rb

@@ -0,0 +1,158 @@
+# Warning: this module is experimental and subject to change and even removal
+# at any time. 
+# 
+# For background/rationale, see:
+#  * http://www.intertwingly.net/blog/2007/01/08/Xhtml5lib
+#  * http://tinyurl.com/ylfj8k (and follow-ups)
+# 
+# References:
+#  * http://googlereader.blogspot.com/2005/12/xml-errors-in-feeds.html
+#  * http://wiki.whatwg.org/wiki/HtmlVsXhtml
+# 
+# @@TODO:
+# * Selectively lowercase only XHTML, but not foreign markup
+require 'html5/html5parser'
+require 'html5/constants'
+
+module HTML5
+
+  # liberal XML parser
+  class XMLParser < HTMLParser
+
+    def initialize(options = {})
+      super options
+      @phases[:initial] = XmlRootPhase.new(self, @tree)
+    end
+
+    def normalize_token(token)
+      case token[:type]
+      when :StartTag, :EmptyTag
+        # We need to remove the duplicate attributes and convert attributes
+        # to a Hash so that [["x", "y"], ["x", "z"]] becomes {"x": "y"}
+
+        token[:data] = Hash[*token[:data].reverse.flatten]
+
+        # For EmptyTags, process both a Start and an End tag
+        if token[:type] == :EmptyTag
+          save = @tokenizer.content_model_flag
+          @phase.processStartTag(token[:name], token[:data])
+          @tokenizer.content_model_flag = save
+          token[:data] = {}
+          token[:type] = :EndTag
+        end
+
+      when :Characters
+        # un-escape RCDATA_ELEMENTS (e.g. style, script)
+        if @tokenizer.content_model_flag == :CDATA
+          token[:data] = token[:data].
+            gsub('&lt;','<').gsub('&gt;','>').gsub('&amp;','&')
+        end
+
+      when :EndTag
+        if token[:data]
+           parse_error(_("End tag contains unexpected attributes."))
+        end
+
+      when :Comment
+        # Rescue CDATA from the comments
+        if token[:data][0..6] == "[CDATA[" and token[:data][-2..-1] == "]]"
+          token[:type] = :Characters
+          token[:data] = token[:data][7 ... -2]
+        end
+      end
+
+      return token
+    end
+  end
+
+  # liberal XMTHML parser
+  class XHTMLParser < XMLParser
+
+    def initialize(options = {})
+      super options
+      @phases[:initial] = InitialPhase.new(self, @tree)
+      @phases[:rootElement] = XhmlRootPhase.new(self, @tree)
+    end
+
+    def normalize_token(token)
+      super(token)
+
+      # ensure that non-void XHTML elements have content so that separate
+      # open and close tags are emitted
+      if token[:type]  == :EndTag
+        if VOID_ELEMENTS.include? token[:name]
+          if @tree.open_elements[-1].name != token["name"]:
+            token[:type] = :EmptyTag
+            token["data"] ||= {}
+          end
+        else
+          if token[:name] == @tree.open_elements[-1].name and \
+            not @tree.open_elements[-1].hasContent
+            @tree.insertText('') unless
+              @tree.open_elements.any? {|e|
+                e.attributes.keys.include? 'xmlns' and
+                e.attributes['xmlns'] != 'http://www.w3.org/1999/xhtml'
+              }
+           end
+        end
+      end
+
+      return token
+    end
+  end
+
+  class XhmlRootPhase < RootElementPhase
+    def insert_html_element
+      element = @tree.createElement("html", {'xmlns' => 'http://www.w3.org/1999/xhtml'})
+      @tree.open_elements.push(element)
+      @tree.document.appendChild(element)
+      @parser.phase = @parser.phases[:beforeHead]
+    end
+  end
+
+  class XmlRootPhase < Phase
+    # Prime the Xml parser
+    @start_tag_handlers = Hash.new(:startTagOther)
+    @end_tag_handlers = Hash.new(:endTagOther)
+    def startTagOther(name, attributes)
+      @tree.open_elements.push(@tree.document)
+      element = @tree.createElement(name, attributes)
+      @tree.open_elements[-1].appendChild(element)
+      @tree.open_elements.push(element)
+      @parser.phase = XmlElementPhase.new(@parser,@tree)
+    end
+    def endTagOther(name)
+      super
+      @tree.open_elements.pop
+    end
+  end
+
+  class XmlElementPhase < Phase
+    # Generic handling for all XML elements
+
+    @start_tag_handlers = Hash.new(:startTagOther)
+    @end_tag_handlers = Hash.new(:endTagOther)
+
+    def startTagOther(name, attributes)
+      element = @tree.createElement(name, attributes)
+      @tree.open_elements[-1].appendChild(element)
+      @tree.open_elements.push(element)
+    end
+
+    def endTagOther(name)
+      for node in @tree.open_elements.reverse
+        if node.name == name
+          {} while @tree.open_elements.pop != node
+          break
+        else
+          parse_error
+        end
+      end
+    end
+
+    def processCharacters(data)
+      @tree.insertText(data)
+    end
+  end
+
+end

data/lib/html5/sanitizer.rb

@@ -0,0 +1,188 @@
+require 'cgi'
+require 'html5/tokenizer'
+
+module HTML5
+
+# This module provides sanitization of XHTML+MathML+SVG
+# and of inline style attributes.
+#
+# It can be either at the Tokenizer stage:
+#
+#       HTMLParser.parse(html, :tokenizer => HTMLSanitizer)
+#
+# or, if you already have a parse tree (in this example, a REXML tree),
+# at the Serializer stage:
+#
+#     tokens = TreeWalkers.get_tree_walker('rexml').new(tree)
+#     HTMLSerializer.serialize(tokens, {:encoding=>'utf-8',
+#        :sanitize => true})
+
+  module HTMLSanitizeModule
+
+    ACCEPTABLE_ELEMENTS = %w[a abbr acronym address area b big blockquote br
+      button caption center cite code col colgroup dd del dfn dir div dl dt
+      em fieldset font form h1 h2 h3 h4 h5 h6 hr i img input ins kbd label
+      legend li map menu ol optgroup option p pre q s samp select small span
+      strike strong sub sup table tbody td textarea tfoot th thead tr tt u
+      ul var]
+
+    MATHML_ELEMENTS = %w[maction math merror mfrac mi mmultiscripts mn mo
+      mover mpadded mphantom mprescripts mroot mrow mspace msqrt mstyle msub
+      msubsup msup mtable mtd mtext mtr munder munderover none]
+
+    SVG_ELEMENTS = %w[a animate animateColor animateMotion animateTransform
+      circle defs desc ellipse font-face font-face-name font-face-src g
+      glyph hkern image linearGradient line marker metadata missing-glyph
+      mpath path polygon polyline radialGradient rect set stop svg switch
+      text title tspan use]
+
+    ACCEPTABLE_ATTRIBUTES = %w[abbr accept accept-charset accesskey action
+      align alt axis border cellpadding cellspacing char charoff charset
+      checked cite class clear cols colspan color compact coords datetime
+      dir disabled enctype for frame headers height href hreflang hspace id
+      ismap label lang longdesc maxlength media method multiple name nohref
+      noshade nowrap prompt readonly rel rev rows rowspan rules scope
+      selected shape size span src start style summary tabindex target title
+      type usemap valign value vspace width xml:lang]
+
+    MATHML_ATTRIBUTES = %w[actiontype align columnalign columnalign
+      columnalign columnlines columnspacing columnspan depth display
+      displaystyle equalcolumns equalrows fence fontstyle fontweight frame
+      height linethickness lspace mathbackground mathcolor mathvariant
+      mathvariant maxsize minsize other rowalign rowalign rowalign rowlines
+      rowspacing rowspan rspace scriptlevel selection separator stretchy
+      width width xlink:href xlink:show xlink:type xmlns xmlns:xlink]
+
+    SVG_ATTRIBUTES = %w[accent-height accumulate additive alphabetic
+       arabic-form ascent attributeName attributeType baseProfile bbox begin
+       by calcMode cap-height class color color-rendering content cx cy d dx
+       dy descent display dur end fill fill-rule font-family font-size
+       font-stretch font-style font-variant font-weight from fx fy g1 g2
+       glyph-name gradientUnits hanging height horiz-adv-x horiz-origin-x id
+       ideographic k keyPoints keySplines keyTimes lang marker-end
+       marker-mid marker-start markerHeight markerUnits markerWidth
+       mathematical max min name offset opacity orient origin
+       overline-position overline-thickness panose-1 path pathLength points
+       preserveAspectRatio r refX refY repeatCount repeatDur
+       requiredExtensions requiredFeatures restart rotate rx ry slope stemh
+       stemv stop-color stop-opacity strikethrough-position
+       strikethrough-thickness stroke stroke-dasharray stroke-dashoffset
+       stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity
+       stroke-width systemLanguage target text-anchor to transform type u1
+       u2 underline-position underline-thickness unicode unicode-range
+       units-per-em values version viewBox visibility width widths x
+       x-height x1 x2 xlink:actuate xlink:arcrole xlink:href xlink:role
+       xlink:show xlink:title xlink:type xml:base xml:lang xml:space xmlns
+       xmlns:xlink y y1 y2 zoomAndPan]
+
+    ATTR_VAL_IS_URI = %w[href src cite action longdesc xlink:href xml:base]
+
+    ACCEPTABLE_CSS_PROPERTIES = %w[azimuth background-color
+      border-bottom-color border-collapse border-color border-left-color
+      border-right-color border-top-color clear color cursor direction
+      display elevation float font font-family font-size font-style
+      font-variant font-weight height letter-spacing line-height overflow
+      pause pause-after pause-before pitch pitch-range richness speak
+      speak-header speak-numeral speak-punctuation speech-rate stress
+      text-align text-decoration text-indent unicode-bidi vertical-align
+      voice-family volume white-space width]
+
+    ACCEPTABLE_CSS_KEYWORDS = %w[auto aqua black block blue bold both bottom
+      brown center collapse dashed dotted fuchsia gray green !important
+      italic left lime maroon medium none navy normal nowrap olive pointer
+      purple red right solid silver teal top transparent underline white
+      yellow]
+
+    ACCEPTABLE_SVG_PROPERTIES = %w[fill fill-opacity fill-rule stroke
+      stroke-width stroke-linecap stroke-linejoin stroke-opacity]
+
+    ACCEPTABLE_PROTOCOLS = %w[ed2k ftp http https irc mailto news gopher nntp
+      telnet webcal xmpp callto feed urn aim rsync tag ssh sftp rtsp afs]
+
+    # subclasses may define their own versions of these constants
+    ALLOWED_ELEMENTS = ACCEPTABLE_ELEMENTS + MATHML_ELEMENTS + SVG_ELEMENTS
+    ALLOWED_ATTRIBUTES = ACCEPTABLE_ATTRIBUTES + MATHML_ATTRIBUTES + SVG_ATTRIBUTES
+    ALLOWED_CSS_PROPERTIES = ACCEPTABLE_CSS_PROPERTIES
+    ALLOWED_CSS_KEYWORDS = ACCEPTABLE_CSS_KEYWORDS
+    ALLOWED_SVG_PROPERTIES = ACCEPTABLE_SVG_PROPERTIES
+    ALLOWED_PROTOCOLS = ACCEPTABLE_PROTOCOLS
+
+    def sanitize_token(token)
+        case token[:type]
+        when :StartTag, :EndTag, :EmptyTag
+          if ALLOWED_ELEMENTS.include?(token[:name])
+            if token.has_key? :data
+              attrs = Hash[*token[:data].flatten]
+              attrs.delete_if { |attr,v| !ALLOWED_ATTRIBUTES.include?(attr) }
+              ATTR_VAL_IS_URI.each do |attr|
+                val_unescaped = CGI.unescapeHTML(attrs[attr].to_s).gsub(/`|[\000-\040\177\s]+|\302[\200-\240]/,'').downcase
+                if val_unescaped =~ /^[a-z0-9][-+.a-z0-9]*:/ and !ALLOWED_PROTOCOLS.include?(val_unescaped.split(':')[0])
+                  attrs.delete attr
+                end
+              end
+              if attrs['style']
+                attrs['style'] = sanitize_css(attrs['style'])
+              end
+              token[:data] = attrs.map {|k,v| [k,v]}
+            end
+            return token
+          else
+            if token[:type] == :EndTag
+              token[:data] = "</#{token[:name]}>"
+            elsif token[:data]
+              attrs = token[:data].map {|k,v| " #{k}=\"#{CGI.escapeHTML(v)}\""}.join('')
+              token[:data] = "<#{token[:name]}#{attrs}>"
+            else
+              token[:data] = "<#{token[:name]}>"
+            end
+            token[:data].insert(-2,'/') if token[:type] == :EmptyTag
+            token[:type] = :Characters
+            token.delete(:name)
+            return token
+          end
+        when :Comment
+          token[:data] = ""
+          return token
+        else
+          return token
+        end
+    end
+
+    def sanitize_css(style)
+      # disallow urls
+      style = style.to_s.gsub(/url\s*\(\s*[^\s)]+?\s*\)\s*/, ' ')
+
+      # gauntlet
+      return '' unless style =~ /^([:,;#%.\sa-zA-Z0-9!]|\w-\w|\'[\s\w]+\'|\"[\s\w]+\"|\([\d,\s]+\))*$/
+      return '' unless style =~ /^(\s*[-\w]+\s*:\s*[^:;]*(;|$))*$/
+
+      clean = []
+      style.scan(/([-\w]+)\s*:\s*([^:;]*)/) do |prop, val|
+        next if val.empty?
+        prop.downcase!
+        if ALLOWED_CSS_PROPERTIES.include?(prop)
+          clean << "#{prop}: #{val};"
+        elsif %w[background border margin padding].include?(prop.split('-')[0])
+          clean << "#{prop}: #{val};" unless val.split().any? do |keyword|
+            !ALLOWED_CSS_KEYWORDS.include?(keyword) and
+            keyword !~ /^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$/
+          end
+        elsif ALLOWED_SVG_PROPERTIES.include?(prop)
+          clean << "#{prop}: #{val};"
+        end
+      end
+
+      style = clean.join(' ')
+    end
+  end
+
+  class HTMLSanitizer < HTMLTokenizer
+    include HTMLSanitizeModule
+    def each
+      super do |token|
+        yield(sanitize_token(token))
+      end
+    end
+  end
+
+end

data/lib/html5/serializer/htmlserializer.rb

@@ -0,0 +1,180 @@
+require 'html5/constants'
+
+module HTML5
+
+  class HTMLSerializer
+
+    def self.serialize(stream, options = {})
+      new(options).serialize(stream, options[:encoding])
+    end
+
+    def escape(string)
+      string.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
+    end
+ 
+    def initialize(options={})
+      @quote_attr_values           = false
+      @quote_char                  = '"'
+      @use_best_quote_char         = true
+      @minimize_boolean_attributes = true
+
+      @use_trailing_solidus          = false
+      @space_before_trailing_solidus = true
+      @escape_lt_in_attrs            = false
+      @escape_rcdata                 = false
+
+      @omit_optional_tags = true
+      @sanitize           = false
+
+      @strip_whitespace = false
+
+      @inject_meta_charset = true
+
+      options.each do |name, value|
+        next unless instance_variable_defined?("@#{name}")
+        @use_best_quote_char = false if name.to_s == 'quote_char'
+        instance_variable_set("@#{name}", value)
+      end
+
+      @errors = []
+    end
+
+    def serialize(treewalker, encoding=nil)
+      in_cdata = false
+      @errors = []
+
+      if encoding and @inject_meta_charset
+        require 'html5/filters/inject_meta_charset'
+        treewalker = Filters::InjectMetaCharset.new(treewalker, encoding)
+      end
+
+      if @strip_whitespace
+        require 'html5/filters/whitespace'
+        treewalker = Filters::WhitespaceFilter.new(treewalker)
+      end
+
+      if @sanitize
+        require 'html5/filters/sanitizer'
+        treewalker = Filters::HTMLSanitizeFilter.new(treewalker)
+      end
+
+      if @omit_optional_tags
+        require 'html5/filters/optionaltags'
+        treewalker = Filters::OptionalTagFilter.new(treewalker)
+      end
+
+      result = []
+      treewalker.each do |token|
+        type = token[:type]
+        if type == :Doctype
+          doctype = "<!DOCTYPE %s>" % token[:name]
+          result << doctype
+
+        elsif [:Characters, :SpaceCharacters].include? type
+          if type == :SpaceCharacters or in_cdata
+            if in_cdata and token[:data].include?("</")
+              serialize_error(_("Unexpected </ in CDATA"))
+            end
+            result << token[:data]
+          else
+            result << escape(token[:data])
+          end
+
+        elsif [:StartTag, :EmptyTag].include? type
+          name = token[:name]
+          if RCDATA_ELEMENTS.include?(name) and not @escape_rcdata
+            in_cdata = true
+          elsif in_cdata
+            serialize_error(_("Unexpected child element of a CDATA element"))
+          end
+          attributes = []
+          for k,v in attrs = token[:data].to_a.sort
+            attributes << ' '
+
+            attributes << k
+            if not @minimize_boolean_attributes or \
+                (!(BOOLEAN_ATTRIBUTES[name]||[]).include?(k) \
+                and !BOOLEAN_ATTRIBUTES[:global].include?(k))
+              attributes << "="
+              if @quote_attr_values or v.empty?
+                quote_attr = true
+              else
+                quote_attr = (SPACE_CHARACTERS + %w(< > " ')).any? {|c| v.include?(c)}
+              end
+              v = v.gsub("&", "&amp;")
+              v = v.gsub("<", "&lt;") if @escape_lt_in_attrs
+              if quote_attr
+                quote_char = @quote_char
+                if @use_best_quote_char
+                  if v.index("'") and !v.index('"')
+                    quote_char = '"'
+                  elsif v.index('"') and !v.index("'")
+                    quote_char = "'"
+                  end
+                end
+                if quote_char == "'"
+                  v = v.gsub("'", "&#39;")
+                else
+                  v = v.gsub('"', "&quot;")
+                end
+                attributes << quote_char << v << quote_char
+              else
+                attributes << v
+              end
+            end
+          end
+          if VOID_ELEMENTS.include?(name) and @use_trailing_solidus
+            if @space_before_trailing_solidus
+              attributes << " /"
+            else
+              attributes << "/"
+            end
+          end
+          result << "<%s%s>" % [name, attributes.join('')]
+
+        elsif type == :EndTag
+          name = token[:name]
+          if RCDATA_ELEMENTS.include?(name)
+            in_cdata = false
+          elsif in_cdata
+            serialize_error(_("Unexpected child element of a CDATA element"))
+          end
+          end_tag = "</#{name}>"
+          result << end_tag
+
+        elsif type == :Comment
+          data = token[:data]
+          serialize_error(_("Comment contains --")) if data.index("--")
+          comment = "<!--%s-->" % token[:data]
+          result << comment
+
+        else
+          serialize_error(token[:data])
+        end
+      end
+
+      if encoding and encoding != 'utf-8'
+        require 'iconv'
+        Iconv.iconv(encoding, 'utf-8', result.join('')).first
+      else
+        result.join('')
+      end
+    end
+
+    alias :render :serialize
+
+    def serialize_error(data="XXX ERROR MESSAGE NEEDED")
+      # XXX The idea is to make data mandatory.
+      @errors.push(data)
+      if @strict
+        raise SerializeError
+      end
+    end
+
+    def _(string); string; end
+  end
+
+  # Error in serialized tree
+  class SerializeError < Exception
+  end
+end

data/lib/html5/serializer/xhtmlserializer.rb

@@ -0,0 +1,20 @@
+require 'html5/serializer/htmlserializer'
+
+module HTML5
+
+  class XHTMLSerializer < HTMLSerializer
+    DEFAULTS = {
+      :quote_attr_values           => true,
+      :minimize_boolean_attributes => false,
+      :use_trailing_solidus        => true,
+      :escape_lt_in_attrs          => true,
+      :omit_optional_tags          => false,
+      :escape_rcdata               => true
+    }
+
+    def initialize(options={})
+      super(DEFAULTS.clone.update(options))
+    end
+  end
+
+end

data/lib/html5/serializer.rb

@@ -0,0 +1,2 @@
+require 'html5/serializer/htmlserializer'
+require 'html5/serializer/xhtmlserializer'