html5 0.1.0

data/lib/html5/html5parser/initial_phase.rb

@@ -0,0 +1,134 @@
+require 'html5/html5parser/phase'
+
+module HTML5
+  class InitialPhase < Phase
+
+    # This phase deals with error handling as well which is currently not
+    # covered in the specification. The error handling is typically known as
+    # "quirks mode". It is expected that a future version of HTML5 will define this.
+
+    def process_eof
+      parse_error(_('Unexpected End of file. Expected DOCTYPE.'))
+      @parser.phase = @parser.phases[:rootElement]
+      @parser.phase.process_eof
+    end
+
+    def processComment(data)
+      @tree.insert_comment(data, @tree.document)
+    end
+
+    def processDoctype(name, publicId, systemId, correct)
+      if name.downcase != 'html' or publicId or systemId
+        parse_error(_('Erroneous DOCTYPE.'))
+      end
+      # XXX need to update DOCTYPE tokens
+      @tree.insertDoctype(name, publicId, systemId)
+
+      publicId = publicId.to_s.upcase
+
+      if name.downcase != 'html'
+        # XXX quirks mode
+      else
+        if ["+//silmaril//dtd html pro v0r11 19970101//en",
+            "-//advasoft ltd//dtd html 3.0 aswedit + extensions//en",
+            "-//as//dtd html 3.0 aswedit + extensions//en",
+            "-//ietf//dtd html 2.0 level 1//en",
+            "-//ietf//dtd html 2.0 level 2//en",
+            "-//ietf//dtd html 2.0 strict level 1//en",
+            "-//ietf//dtd html 2.0 strict level 2//en",
+            "-//ietf//dtd html 2.0 strict//en",
+            "-//ietf//dtd html 2.0//en",
+            "-//ietf//dtd html 2.1e//en",
+            "-//ietf//dtd html 3.0//en",
+            "-//ietf//dtd html 3.0//en//",
+            "-//ietf//dtd html 3.2 final//en",
+            "-//ietf//dtd html 3.2//en",
+            "-//ietf//dtd html 3//en",
+            "-//ietf//dtd html level 0//en",
+            "-//ietf//dtd html level 0//en//2.0",
+            "-//ietf//dtd html level 1//en",
+            "-//ietf//dtd html level 1//en//2.0",
+            "-//ietf//dtd html level 2//en",
+            "-//ietf//dtd html level 2//en//2.0",
+            "-//ietf//dtd html level 3//en",
+            "-//ietf//dtd html level 3//en//3.0",
+            "-//ietf//dtd html strict level 0//en",
+            "-//ietf//dtd html strict level 0//en//2.0",
+            "-//ietf//dtd html strict level 1//en",
+            "-//ietf//dtd html strict level 1//en//2.0",
+            "-//ietf//dtd html strict level 2//en",
+            "-//ietf//dtd html strict level 2//en//2.0",
+            "-//ietf//dtd html strict level 3//en",
+            "-//ietf//dtd html strict level 3//en//3.0",
+            "-//ietf//dtd html strict//en",
+            "-//ietf//dtd html strict//en//2.0",
+            "-//ietf//dtd html strict//en//3.0",
+            "-//ietf//dtd html//en",
+            "-//ietf//dtd html//en//2.0",
+            "-//ietf//dtd html//en//3.0",
+            "-//metrius//dtd metrius presentational//en",
+            "-//microsoft//dtd internet explorer 2.0 html strict//en",
+            "-//microsoft//dtd internet explorer 2.0 html//en",
+            "-//microsoft//dtd internet explorer 2.0 tables//en",
+            "-//microsoft//dtd internet explorer 3.0 html strict//en",
+            "-//microsoft//dtd internet explorer 3.0 html//en",
+            "-//microsoft//dtd internet explorer 3.0 tables//en",
+            "-//netscape comm. corp.//dtd html//en",
+            "-//netscape comm. corp.//dtd strict html//en",
+            "-//o'reilly and associates//dtd html 2.0//en",
+            "-//o'reilly and associates//dtd html extended 1.0//en",
+            "-//spyglass//dtd html 2.0 extended//en",
+            "-//sq//dtd html 2.0 hotmetal + extensions//en",
+            "-//sun microsystems corp.//dtd hotjava html//en",
+            "-//sun microsystems corp.//dtd hotjava strict html//en",
+            "-//w3c//dtd html 3 1995-03-24//en",
+            "-//w3c//dtd html 3.2 draft//en",
+            "-//w3c//dtd html 3.2 final//en",
+            "-//w3c//dtd html 3.2//en",
+            "-//w3c//dtd html 3.2s draft//en",
+            "-//w3c//dtd html 4.0 frameset//en",
+            "-//w3c//dtd html 4.0 transitional//en",
+            "-//w3c//dtd html experimental 19960712//en",
+            "-//w3c//dtd html experimental 970421//en",
+            "-//w3c//dtd w3 html//en",
+            "-//w3o//dtd w3 html 3.0//en",
+            "-//w3o//dtd w3 html 3.0//en//",
+            "-//w3o//dtd w3 html strict 3.0//en//",
+            "-//webtechs//dtd mozilla html 2.0//en",
+            "-//webtechs//dtd mozilla html//en",
+            "-/w3c/dtd html 4.0 transitional/en",
+            "html"].include?(publicId) or
+          (systemId == nil and
+            ["-//w3c//dtd html 4.01 frameset//EN",
+             "-//w3c//dtd html 4.01 transitional//EN"].include?(publicId)) or
+          (systemId == "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd")
+            #XXX quirks mode
+          end
+      end
+
+      @parser.phase = @parser.phases[:rootElement]
+    end
+
+    def processSpaceCharacters(data)
+    end
+
+    def processCharacters(data)
+      parse_error(_('Unexpected non-space characters. Expected DOCTYPE.'))
+      @parser.phase = @parser.phases[:rootElement]
+      @parser.phase.processCharacters(data)
+    end
+
+    def processStartTag(name, attributes)
+      parse_error(_("Unexpected start tag (#{name}). Expected DOCTYPE."))
+      @parser.phase = @parser.phases[:rootElement]
+      @parser.phase.processStartTag(name, attributes)
+    end
+
+    def processEndTag(name)
+      parse_error(_("Unexpected end tag (#{name}). Expected DOCTYPE."))
+      @parser.phase = @parser.phases[:rootElement]
+      @parser.phase.processEndTag(name)
+    end
+
+  end
+end

data/lib/html5/html5parser/phase.rb

@@ -0,0 +1,158 @@
+module HTML5
+  # Base class for helper objects that implement each phase of processing.
+  #
+  # Handler methods should be in the following order (they can be omitted):
+  #
+  #   * EOF
+  #   * Comment
+  #   * Doctype
+  #   * SpaceCharacters
+  #   * Characters
+  #   * StartTag
+  #     - startTag* methods
+  #   * EndTag
+  #     - endTag* methods
+  #
+  class Phase
+
+    extend Forwardable
+    def_delegators :@parser, :parse_error
+
+    # The following example call:
+    #
+    #   tag_handlers('startTag', 'html', %w( base link meta ), %w( li dt dd ) => 'ListItem')
+    #
+    # ...would return a hash equal to this:
+    #
+    #   { 'html' => 'startTagHtml',
+    #     'base' => 'startTagBaseLinkMeta',
+    #     'link' => 'startTagBaseLinkMeta',
+    #     'meta' => 'startTagBaseLinkMeta',
+    #     'li'   => 'startTagListItem',
+    #     'dt'   => 'startTagListItem',
+    #     'dd'   => 'startTagListItem'  }
+    #
+    def self.tag_handlers(prefix, *tags)
+      mapping = {}
+      if tags.last.is_a?(Hash)
+        tags.pop.each do |names, handler_method_suffix|
+          handler_method = prefix + handler_method_suffix
+          Array(names).each {|name| mapping[name] = handler_method }
+        end
+      end
+      tags.each do |names|
+        names = Array(names)
+        handler_method = prefix + names.map {|name| name.capitalize }.join
+        names.each {|name| mapping[name] = handler_method }
+      end
+      mapping
+    end
+
+    def self.start_tag_handlers
+      @start_tag_handlers ||= Hash.new('startTagOther')
+    end
+
+    # Declare what start tags this Phase handles. Can be called more than once.
+    #
+    # Example usage:
+    #
+    #   handle_start 'html'
+    #   # html start tags will be handled by a method named 'startTagHtml'
+    #
+    #   handle_start %( base link meta )
+    #   # base, link and meta start tags will be handled by a method named 'startTagBaseLinkMeta'
+    #
+    #   handle_start %( li dt dd ) => 'ListItem'
+    #   # li, dt, and dd start tags will be handled by a method named 'startTagListItem'
+    #
+    def self.handle_start(*tags)
+      start_tag_handlers.update tag_handlers('startTag', *tags)
+    end
+
+    def self.end_tag_handlers
+      @end_tag_handlers ||= Hash.new('endTagOther')
+    end
+
+    # Declare what end tags this Phase handles. Behaves like handle_start.
+    #
+    def self.handle_end(*tags)
+      end_tag_handlers.update tag_handlers('endTag', *tags)
+    end
+
+    def initialize(parser, tree)
+      @parser, @tree = parser, tree
+    end
+
+    def process_eof
+      @tree.generateImpliedEndTags
+
+      if @tree.open_elements.length > 2
+        parse_error(_('Unexpected end of file. Missing closing tags.'))
+      elsif @tree.open_elements.length == 2 and @tree.open_elements[1].name != 'body'
+        # This happens for framesets or something?
+        parse_error(_("Unexpected end of file. Expected end tag (#{@tree.open_elements[1].name}) first."))
+      elsif @parser.inner_html and @tree.open_elements.length > 1 
+        # XXX This is not what the specification says. Not sure what to do here.
+        parse_error(_('XXX inner_html EOF'))
+      end
+      # Betting ends.
+    end
+
+    def processComment(data)
+      # For most phases the following is correct. Where it's not it will be
+      # overridden.
+      @tree.insert_comment(data, @tree.open_elements.last)
+    end
+
+    def processDoctype(name, publicId, systemId, correct)
+      parse_error(_('Unexpected DOCTYPE. Ignored.'))
+    end
+
+    def processSpaceCharacters(data)
+      @tree.insertText(data)
+    end
+
+    def processStartTag(name, attributes)
+      send self.class.start_tag_handlers[name], name, attributes
+    end
+
+    def startTagHtml(name, attributes)
+      if @parser.first_start_tag == false and name == 'html'
+         parse_error(_('html needs to be the first start tag.'))
+      end
+      # XXX Need a check here to see if the first start tag token emitted is
+      # this token... If it's not, invoke parse_error.
+      attributes.each do |attr, value|
+        unless @tree.open_elements.first.attributes.has_key?(attr)
+          @tree.open_elements.first.attributes[attr] = value
+        end
+      end
+      @parser.first_start_tag = false
+    end
+
+    def processEndTag(name)
+      send self.class.end_tag_handlers[name], name
+    end
+
+    def _(string)
+      string
+    end
+
+    def assert(value)
+      throw AssertionError.new unless value
+    end
+
+    def in_scope?(*args)
+      @tree.elementInScope(*args)
+    end
+
+    def remove_open_elements_until(name=nil)
+      finished = false
+      until finished
+        element = @tree.open_elements.pop
+        finished = name.nil? ? yield(element) : element.name == name
+      end
+      return element
+    end
+  end
+end

data/lib/html5/html5parser/root_element_phase.rb

@@ -0,0 +1,42 @@
+require 'html5/html5parser/phase'
+
+module HTML5
+  class RootElementPhase < Phase
+
+    def process_eof
+      insert_html_element
+      @parser.phase.process_eof
+    end
+
+    def processComment(data)
+      @tree.insert_comment(data, @tree.document)
+    end
+
+    def processSpaceCharacters(data)
+    end
+
+    def processCharacters(data)
+      insert_html_element
+      @parser.phase.processCharacters(data)
+    end
+
+    def processStartTag(name, attributes)
+      @parser.first_start_tag = true if name == 'html'
+      insert_html_element
+      @parser.phase.processStartTag(name, attributes)
+    end
+
+    def processEndTag(name)
+      insert_html_element
+      @parser.phase.processEndTag(name)
+    end
+
+    def insert_html_element
+      element = @tree.createElement('html', {})
+      @tree.open_elements.push(element)
+      @tree.document.appendChild(element)
+      @parser.phase = @parser.phases[:beforeHead]
+    end
+
+  end
+end

data/lib/html5/html5parser/trailing_end_phase.rb

@@ -0,0 +1,35 @@
+require 'html5/html5parser/phase'
+
+module HTML5
+  class TrailingEndPhase < Phase
+
+    def process_eof
+    end
+
+    def processComment(data)
+      @tree.insert_comment(data, @tree.document)
+    end
+
+    def processSpaceCharacters(data)
+      @parser.last_phase.processSpaceCharacters(data)
+    end
+
+    def processCharacters(data)
+      parse_error(_('Unexpected non-space characters. Expected end of file.'))
+      @parser.phase = @parser.last_phase
+      @parser.phase.processCharacters(data)
+    end
+
+    def processStartTag(name, attributes)
+      parse_error(_('Unexpected start tag (#{name}). Expected end of file.'))
+      @parser.phase = @parser.last_phase
+      @parser.phase.processStartTag(name, attributes)
+    end
+
+    def processEndTag(name)
+      parse_error(_('Unexpected end tag (#{name}). Expected end of file.'))
+      @parser.phase = @parser.last_phase
+      @parser.phase.processEndTag(name)
+    end
+  end
+end

data/lib/html5/html5parser.rb

@@ -0,0 +1,248 @@
+require 'html5/constants'
+require 'html5/tokenizer'
+require 'html5/treebuilders/rexml'
+
+Dir.glob(File.join(File.dirname(__FILE__), 'html5parser', '*_phase.rb')).each do |path|
+  require 'html5/html5parser/' + File.basename(path)
+end
+
+module HTML5
+
+  # Error in parsed document
+  class ParseError < Exception; end
+  class AssertionError < Exception; end
+
+  # HTML parser. Generates a tree structure from a stream of (possibly malformed) HTML
+  #
+  class HTMLParser
+
+    attr_accessor :phase, :first_start_tag, :inner_html, :last_phase, :insert_from_table
+
+    attr_reader :phases, :tokenizer, :tree, :errors
+
+    def self.parse(stream, options = {})
+      encoding = options.delete(:encoding)
+      new(options).parse(stream,encoding)
+    end
+
+    def self.parse_fragment(stream, options = {})
+      container = options.delete(:container) || 'div'
+      encoding = options.delete(:encoding)
+      new(options).parse_fragment(stream, container, encoding)
+    end
+
+    @@phases = %w( initial rootElement beforeHead inHead afterHead inBody inTable inCaption
+      inColumnGroup inTableBody inRow inCell inSelect afterBody inFrameset afterFrameset trailingEnd )
+
+    # :strict - raise an exception when a parse error is encountered
+    # :tree - a treebuilder class controlling the type of tree that will be
+    # returned. Built in treebuilders can be accessed through
+    # HTML5::TreeBuilders[treeType]
+    def initialize(options = {})
+      @strict = false
+      @errors = []
+     
+      @tokenizer =  HTMLTokenizer
+      @tree = TreeBuilders::REXML::TreeBuilder
+
+      options.each {|name, value| instance_variable_set("@#{name}", value) }
+      @lowercase_attr_name    = nil unless instance_variable_defined?(:@lowercase_attr_name)
+      @lowercase_element_name = nil unless instance_variable_defined?(:@lowercase_element_name)
+
+      @tree = @tree.new
+
+      @phases = @@phases.inject({}) do |phases, phase_name|
+        phase_class_name = phase_name.sub(/(.)/) { $1.upcase } + 'Phase'
+        phases[phase_name.to_sym] = HTML5.const_get(phase_class_name).new(self, @tree)
+        phases
+      end
+    end
+
+    def _parse(stream, inner_html, encoding, container = 'div')
+      @tree.reset
+      @first_start_tag = false
+      @errors = []
+
+      @tokenizer = @tokenizer.class unless Class === @tokenizer
+      @tokenizer = @tokenizer.new(stream, :encoding => encoding,
+        :parseMeta => !inner_html, :lowercase_attr_name => @lowercase_attr_name, :lowercase_element_name => @lowercase_element_name)
+
+      if inner_html
+        case @inner_html = container.downcase
+          when 'title', 'textarea'
+            @tokenizer.content_model_flag = :RCDATA
+          when 'style', 'script', 'xmp', 'iframe', 'noembed', 'noframes', 'noscript'
+            @tokenizer.content_model_flag = :CDATA
+          when 'plaintext'
+            @tokenizer.content_model_flag = :PLAINTEXT
+          else
+          # content_model_flag already is PCDATA
+          #@tokenizer.content_model_flag = :PCDATA
+        end
+      
+        @phase = @phases[:rootElement]
+        @phase.insert_html_element
+        reset_insertion_mode
+      else
+        @inner_html = false
+        @phase = @phases[:initial]
+      end
+
+      # We only seem to have InBodyPhase testcases where the following is
+      # relevant ... need others too
+      @last_phase = nil
+
+      # XXX This is temporary for the moment so there isn't any other
+      # changes needed for the parser to work with the iterable tokenizer
+      @tokenizer.each do |token|
+        token = normalize_token(token)
+
+        method = 'process%s' % token[:type]
+
+        case token[:type]
+          when :Characters, :SpaceCharacters, :Comment
+            @phase.send method, token[:data]
+          when :StartTag
+            @phase.send method, token[:name], token[:data]
+          when :EndTag
+            @phase.send method, token[:name]
+          when :Doctype
+            @phase.send method, token[:name], token[:publicId],
+              token[:systemId], token[:correct]
+          else
+            parse_error(token[:data])
+        end
+      end
+
+      # When the loop finishes it's EOF
+      @phase.process_eof
+    end
+
+    # Parse a HTML document into a well-formed tree
+    #
+    # stream - a filelike object or string containing the HTML to be parsed
+    #
+    # The optional encoding parameter must be a string that indicates
+    # the encoding.  If specified, that encoding will be used,
+    # regardless of any BOM or later declaration (such as in a meta
+    # element)
+    def parse(stream, encoding=nil)
+      _parse(stream, false, encoding)
+      @tree.get_document
+    end
+
+    # Parse a HTML fragment into a well-formed tree fragment
+
+    # container - name of the element we're setting the inner_html property
+    # if set to nil, default to 'div'
+    #
+    # stream - a filelike object or string containing the HTML to be parsed
+    #
+    # The optional encoding parameter must be a string that indicates
+    # the encoding.  If specified, that encoding will be used,
+    # regardless of any BOM or later declaration (such as in a meta
+    # element)
+    def parse_fragment(stream, container='div', encoding=nil)
+      _parse(stream, true, encoding, container)
+      @tree.get_fragment
+    end
+
+    def parse_error(data = 'XXX ERROR MESSAGE NEEDED')
+      # XXX The idea is to make data mandatory.
+      @errors.push([@tokenizer.stream.position, data])
+      raise ParseError if @strict
+    end
+
+    # HTML5 specific normalizations to the token stream
+    def normalize_token(token)
+
+      if token[:type] == :EmptyTag
+        # When a solidus (/) is encountered within a tag name what happens
+        # depends on whether the current tag name matches that of a void
+        # element.  If it matches a void element atheists did the wrong
+        # thing and if it doesn't it's wrong for everyone.
+
+        unless VOID_ELEMENTS.include?(token[:name])
+          parse_error(_('Solidus (/) incorrectly placed in tag.'))
+        end
+
+        token[:type] = :StartTag
+      end
+
+      if token[:type] == :StartTag
+        token[:name] = token[:name].downcase
+
+        # We need to remove the duplicate attributes and convert attributes
+        # to a dict so that [["x", "y"], ["x", "z"]] becomes {"x": "y"}
+
+        unless token[:data].empty?
+          data = token[:data].reverse.map {|attr, value| [attr.downcase, value] }
+          token[:data] = Hash[*data.flatten]
+        end
+
+      elsif token[:type] == :EndTag
+        parse_error(_('End tag contains unexpected attributes.')) unless token[:data].empty?
+        token[:name] = token[:name].downcase
+      end
+
+      token
+    end
+
+    @@new_modes = {
+      'select'   => :inSelect,
+      'td'       => :inCell,
+      'th'       => :inCell,
+      'tr'       => :inRow,
+      'tbody'    => :inTableBody,
+      'thead'    => :inTableBody,
+      'tfoot'    => :inTableBody,
+      'caption'  => :inCaption,
+      'colgroup' => :inColumnGroup,
+      'table'    => :inTable,
+      'head'     => :inBody,
+      'body'     => :inBody,
+      'frameset' => :inFrameset
+    }
+
+    def reset_insertion_mode
+      # The name of this method is mostly historical. (It's also used in the
+      # specification.)
+      last = false
+
+      @tree.open_elements.reverse.each do |node|
+        node_name = node.name
+
+        if node == @tree.open_elements.first
+          last = true
+          unless ['td', 'th'].include?(node_name)
+            # XXX
+            # assert @inner_html
+            node_name = @inner_html
+          end
+        end
+
+        # Check for conditions that should only happen in the inner_html
+        # case
+        if ['select', 'colgroup', 'head', 'frameset'].include?(node_name)
+          # XXX
+          # assert @inner_html
+        end
+
+        if @@new_modes.has_key?(node_name)
+          @phase = @phases[@@new_modes[node_name]]
+        elsif node_name == 'html'
+          @phase = @phases[@tree.head_pointer.nil?? :beforeHead : :afterHead]
+        elsif last
+          @phase = @phases[:inBody]
+        else
+          next
+        end
+
+        break
+      end
+    end
+
+    def _(string); string; end
+  end
+
+end