html5 0.1.0

data/lib/html5/treewalkers.rb

@@ -0,0 +1,26 @@
+require 'html5/treewalkers/base'
+
+module HTML5
+  module TreeWalkers
+
+    class << self
+      def [](name)
+        case name.to_s.downcase
+        when 'simpletree'
+          require 'html5/treewalkers/simpletree'
+          SimpleTree::TreeWalker
+        when 'rexml'
+          require 'html5/treewalkers/rexml'
+          REXML::TreeWalker
+        when 'hpricot'
+          require 'html5/treewalkers/hpricot'
+          Hpricot::TreeWalker
+        else
+          raise "Unknown TreeWalker #{name}"
+        end
+      end
+
+      alias :get_tree_walker :[]
+    end
+  end
+end

data/lib/html5.rb

@@ -0,0 +1,13 @@
+require 'html5/html5parser'
+require 'html5/version'
+
+module HTML5
+
+  def self.parse(stream, options={})
+    HTMLParser.parse(stream, options)
+  end
+
+  def self.parse_fragment(stream, options={})
+    HTMLParser.parse(stream, options)
+  end
+end

data/parse.rb

@@ -0,0 +1,217 @@
+#!/usr/bin/env ruby
+# 
+# Parse a document to a simpletree tree, with optional profiling
+
+$:.unshift File.dirname(__FILE__),'lib'
+
+def parse(opts, args)
+  encoding = nil
+
+  f = args[-1]
+  if f
+    begin
+      if f[0..6] == 'http://'
+        require 'open-uri'
+        f = URI.parse(f).open
+        encoding = f.charset
+      elsif f == '-'
+        f = $stdin
+      else
+        f = open(f)
+      end
+    rescue
+    end
+  else
+    $stderr.write("No filename provided. Use -h for help\n")
+    exit(1)
+  end
+
+  require 'html5/treebuilders'
+  treebuilder = HTML5::TreeBuilders[opts.treebuilder]
+
+  if opts.output == :xml
+    require 'html5/liberalxmlparser'
+    p = HTML5::XHTMLParser.new(:tree=>treebuilder)
+  else
+    require 'html5/html5parser'
+    p = HTML5::HTMLParser.new(:tree=>treebuilder)
+  end
+
+  if opts.parsemethod == :parse
+    args = [f, encoding]
+  else
+    args = [f, 'div', encoding]
+  end
+
+  if opts.profile
+    require 'profiler'
+    Profiler__::start_profile
+    p.send(opts.parsemethod, *args)
+    Profiler__::stop_profile
+    Profiler__::print_profile($stderr)
+  elsif opts.time
+    require 'time' # TODO: switch to benchmark
+    t0 = Time.new
+    document = p.send(opts.parsemethod, *args)
+    t1 = Time.new
+    printOutput(p, document, opts)
+    t2 = Time.new
+    puts "\n\nRun took: %fs (plus %fs to print the output)"%[t1-t0, t2-t1]
+  else
+    document = p.send(opts.parsemethod, *args)
+    printOutput(p, document, opts)
+  end
+end
+
+def printOutput(parser, document, opts)
+  puts "Encoding: #{parser.tokenizer.stream.char_encoding}" if opts.encoding
+
+  case opts.output
+  when :xml
+    print document
+  when :html
+    require 'html5/treewalkers'
+    tokens = HTML5::TreeWalkers[opts.treebuilder].new(document)
+    require 'html5/serializer'
+    puts HTML5::HTMLSerializer.serialize(tokens, opts.serializer)
+  when :hilite
+    print document.hilite
+  when :tree
+    document = [document] unless document.respond_to?(:each)
+    document.each {|fragment| puts parser.tree.testSerializer(fragment)}
+  end
+
+  if opts.error
+    errList=[]
+    for pos, message in parser.errors
+        errList << ("Line %i Col %i"%pos + " " + message)
+    end
+    $stdout.write("\nParse errors:\n" + errList.join("\n")+"\n")
+  end
+end
+
+require 'ostruct'
+options = OpenStruct.new
+options.profile = false
+options.time = false
+options.output = :html
+options.treebuilder = 'simpletree'
+options.error = false
+options.encoding = false
+options.parsemethod = :parse
+options.serializer = {
+  :encoding => 'utf-8',
+  :omit_optional_tags => false,
+  :inject_meta_charset => false
+}
+
+require 'optparse'
+opts = OptionParser.new do |opts|
+  opts.separator ""
+  opts.separator "Parse Options:"
+
+  opts.on("-b", "--treebuilder NAME") do |treebuilder|
+    options.treebuilder = treebuilder
+  end
+
+  opts.on("-f", "--fragment", "Parse as a fragment") do |parse|
+    options.parsemethod = :parse_fragment
+  end
+
+  opts.separator ""
+  opts.separator "Filter Options:"
+
+  opts.on("--[no-]inject-meta-charset", "inject <meta charset>") do |inject|
+    options.serializer[:inject_meta_charset] = inject
+  end
+
+  opts.on("--[no-]strip-whitespace", "strip unnecessary whitespace") do |strip|
+    options.serializer[:strip_whitespace] = strip
+  end
+
+  opts.on("--[no-]sanitize", "escape unsafe tags") do |sanitize|
+    options.serializer[:sanitize] = sanitize
+  end
+
+  opts.separator ""
+  opts.separator "Output Options:"
+
+  opts.on("--tree", "output as debug tree") do |tree|
+    options.output = :tree
+  end
+  
+  opts.on("-x", "--xml", "output as xml") do |xml|
+    options.output = :xml
+    options.treebuilder = "rexml"
+  end
+  
+  opts.on("--[no-]html", "Output as html") do |html|
+    options.output = (html ? :html : nil)
+  end
+  
+  opts.on("--hilite", "Output as formatted highlighted code.") do |hilite|
+    options.output = :hilite
+  end
+  
+  opts.on("-e", "--error", "Print a list of parse errors") do |error|
+    options.error = error
+  end
+
+  opts.separator ""
+  opts.separator "Serialization Options:"
+
+  opts.on("--[no-]omit-optional-tags", "Omit optional tags") do |omit|
+    options.serializer[:omit_optional_tags] = omit
+  end
+
+  opts.on("--[no-]quote-attr-values", "Quote attribute values") do |quote|
+    options.serializer[:quote_attr_values] = quote
+  end
+
+  opts.on("--[no-]use-best-quote-char", "Use best quote character") do |best|
+    options.serializer[:use_best_quote_char] = best
+  end
+
+  opts.on("--quote-char C", "Use specified quote character") do |c|
+    options.serializer[:quote_char] = c
+  end
+
+  opts.on("--[no-]minimize-boolean-attributes", "Minimize boolean attributes") do |min|
+    options.serializer[:minimize_boolean_attributes] = min
+  end
+
+  opts.on("--[no-]use-trailing-solidus", "Use trailing solidus") do |slash|
+    options.serializer[:use_trailing_solidus] = slash
+  end
+
+  opts.on("--[no-]escape-lt-in-attrs", "Escape less than signs in attribute values") do |lt|
+    options.serializer[:escape_lt_in_attrs] = lt
+  end
+
+  opts.on("--[no-]escape-rcdata", "Escape rcdata element values") do |rcdata|
+    options.serializer[:escape_rcdata] = rcdata
+  end
+
+  opts.separator ""
+  opts.separator "Other Options:"
+
+  opts.on("-p", "--[no-]profile", "Profile the run") do |profile|
+    options.profile = profile
+  end
+    
+  opts.on("-t", "--[no-]time", "Time the run") do |time|
+    options.time = time
+  end
+    
+  opts.on("-c", "--[no-]encoding", "Print character encoding used") do |encoding|
+    options.encoding = encoding
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end
+
+opts.parse!(ARGV)
+parse options, ARGV

data/tests/preamble.rb

@@ -0,0 +1,82 @@
+require 'test/unit'
+
+HTML5_BASE = File.dirname(File.dirname(File.dirname(File.expand_path(__FILE__)))) 
+
+if File.exists?(File.join(HTML5_BASE, 'testdata'))
+  TESTDATA_DIR = File.join(HTML5_BASE, 'testdata')
+else
+  TESTDATA_DIR = File.join(File.dirname(File.dirname(File.expand_path(__FILE__))), 'testdata')
+end
+
+$:.unshift File.join(File.dirname(File.dirname(__FILE__)),'lib')
+
+$:.unshift File.dirname(__FILE__)
+
+def html5_test_files(subdirectory)
+  Dir[File.join(TESTDATA_DIR, subdirectory, '*.*')]
+end
+
+begin
+  require 'rubygems'
+  require 'json'
+rescue LoadError
+  class JSON
+    def self.parse json
+      json.gsub!(/\$/, "\\$")
+      json.gsub!(/"\s*:/, '"=>')
+      json.gsub!(/\\u[0-9a-fA-F]{4}/) {|x| [x[2..-1].to_i(16)].pack('U')}
+      null = nil
+      eval json
+    end
+  end
+end
+
+module HTML5
+  module TestSupport
+    # convert the output of str(document) to the format used in the testcases
+    def convertTreeDump(treedump)
+      treedump.split(/\n/)[1..-1].map { |line| (line.length > 2 and line[0] == ?|) ? line[3..-1] : line }.join("\n")
+    end
+
+    def sortattrs(output)
+      output.gsub(/^(\s+)\w+=.*(\n\1\w+=.*)+/) do |match|
+         match.split("\n").sort.join("\n")
+      end
+    end
+
+    class TestData
+      include Enumerable
+
+      def initialize(filename, sections)
+        @f = open(filename)
+        @sections = sections
+      end
+    
+      def each
+        data = {}
+        key=nil
+        @f.each_line do |line|
+          if line[0] == ?# and @sections.include?(line[1..-2])
+            heading = line[1..-2]
+            if data.any? and heading == @sections[0]
+              data[key].chomp!  #Remove trailing newline
+              yield normaliseOutput(data)
+              data = {}
+            end
+            key = heading
+            data[key]=""
+          elsif key
+            data[key] += line
+          end
+        end
+        yield normaliseOutput(data) if data
+      end
+        
+      def normaliseOutput(data)
+        #Remove trailing newlines
+        data.keys.each { |key| data[key].chomp! }
+        @sections.map {|heading| data[heading]}
+      end
+    end
+  end
+end

data/tests/test_encoding.rb

@@ -0,0 +1,35 @@
+require File.join(File.dirname(__FILE__), 'preamble')
+
+require 'html5/inputstream'
+
+class Html5EncodingTestCase < Test::Unit::TestCase
+  include HTML5
+  include TestSupport
+
+  begin
+    require 'rubygems'
+    require 'UniversalDetector'
+
+    def test_chardet
+      file = File.open(File.join(TESTDATA_DIR, 'encoding', 'chardet', 'test_big5.txt'), 'r')
+      stream = HTML5::HTMLInputStream.new(file, :chardet => true)
+      assert_equal 'big5', stream.char_encoding.downcase
+    rescue LoadError
+      puts "chardet not found, skipping chardet tests"
+    end
+  end
+
+  html5_test_files('encoding').each do |test_file|        
+    test_name = File.basename(test_file).sub('.dat', '').tr('-', '')
+
+    TestData.new(test_file, %w(data encoding)).
+      each_with_index do |(input, encoding), index|
+
+      define_method 'test_%s_%d' % [ test_name, index + 1 ] do
+        stream = HTML5::HTMLInputStream.new(input, :chardet => false)
+        assert_equal encoding.downcase, stream.char_encoding.downcase, input
+      end
+    end
+  end
+
+end

data/tests/test_lxp.rb

@@ -0,0 +1,263 @@
+require File.join(File.dirname(__FILE__), 'preamble')
+
+require 'html5/liberalxmlparser'
+
+XMLELEM = /<(\w+\s*)((?:[-:\w]+="[^"]*"\s*)+)(\/?)>/
+
+def assert_xml_equal(input, expected=nil, parser=HTML5::XMLParser)
+  sortattrs = proc {"<#{$1+$2.split.sort.join(' ')+$3}>"}
+  document = parser.parse(input.chomp, :lowercase_attr_name => false, :lowercase_element_name => false).root
+  if not expected
+    expected = input.chomp.gsub(XMLELEM,&sortattrs)
+    expected = expected.gsub(/&#(\d+);/) {[$1.to_i].pack('U')}
+    output = document.to_s.gsub(/'/,'"').gsub(XMLELEM,&sortattrs)
+    assert_equal(expected, output)
+  else
+    assert_equal(expected, document.to_s.gsub(/'/,'"'))
+  end
+end
+
+def assert_xhtml_equal(input, expected=nil, parser=HTML5::XHTMLParser)
+  assert_xml_equal(input, expected, parser)
+end
+
+class BasicXhtml5Test < Test::Unit::TestCase
+
+  def test_title_body_mismatched_close
+    assert_xhtml_equal(
+      '<title>Xhtml</title><b><i>content</b></i>',
+      '<html xmlns="http://www.w3.org/1999/xhtml">' +
+      '<head><title>Xhtml</title></head>' + 
+      '<body><b><i>content</i></b></body>' +
+      '</html>')
+  end
+
+  def test_title_body_named_charref
+    assert_xhtml_equal(
+      '<title>ntilde</title>A &ntilde B',
+      '<html xmlns="http://www.w3.org/1999/xhtml">' +
+      '<head><title>ntilde</title></head>' + 
+      '<body>A '+ [0xF1].pack('U') + ' B</body>' +
+      '</html>')
+  end
+end
+
+class BasicXmlTest < Test::Unit::TestCase
+
+  def test_comment
+    assert_xml_equal("<x><!-- foo --></x>")
+  end
+
+  def test_cdata
+    assert_xml_equal("<x><![CDATA[foo]]></x>","<x>foo</x>")
+  end
+
+  def test_simple_text
+    assert_xml_equal("<p>foo</p>","<p>foo</p>")
+  end
+
+  def test_optional_close
+    assert_xml_equal("<p>foo","<p>foo</p>")
+  end
+
+  def test_html_mismatched
+    assert_xml_equal("<b><i>foo</b></i>","<b><i>foo</i></b>")
+  end
+end
+
+class OpmlTest < Test::Unit::TestCase
+
+  def test_mixedCaseElement
+    assert_xml_equal(
+      '<opml version="1.0">' +
+      '<head><ownerName>Dave Winer</ownerName></head>' +
+      '</opml>')
+  end
+
+  def test_mixedCaseAttribute
+    assert_xml_equal(
+      '<opml version="1.0">' +
+      '<body><outline isComment="true"/></body>' +
+      '</opml>')
+  end
+
+  def test_malformed
+    assert_xml_equal(
+      '<opml version="1.0">' +
+      '<body><outline text="Odds & Ends"/></body>' +
+      '</opml>',
+      '<opml version="1.0">' +
+      '<body><outline text="Odds &amp; Ends"/></body>' +
+      '</opml>')
+  end
+end
+
+class XhtmlTest < Test::Unit::TestCase
+
+  def test_mathml
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>MathML</title></head>
+<body>
+  <math xmlns="http://www.w3.org/1998/Math/MathML">
+  <mrow>
+    <mi>x</mi>
+    <mo>=</mo>
+
+    <mfrac>
+    <mrow>
+      <mrow>
+      <mo>-</mo>
+      <mi>b</mi>
+      </mrow>
+      <mo>&#177;</mo>
+      <msqrt>
+
+      <mrow>
+        <msup>
+        <mi>b</mi>
+        <mn>2</mn>
+        </msup>
+        <mo>-</mo>
+        <mrow>
+
+        <mn>4</mn>
+        <mo>&#8290;</mo>
+        <mi>a</mi>
+        <mo>&#8290;</mo>
+        <mi>c</mi>
+        </mrow>
+      </mrow>
+
+      </msqrt>
+    </mrow>
+    <mrow>
+      <mn>2</mn>
+      <mo>&#8290;</mo>
+      <mi>a</mi>
+    </mrow>
+    </mfrac>
+
+  </mrow>
+  </math>
+</body></html>
+EOX
+  end
+
+  def test_svg
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>SVG</title></head>
+<body>
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
+  <path d="M38,38c0-12,24-15,23-2c0,9-16,13-16,23v7h11v-4c0-9,17-12,17-27
+       c-2-22-45-22-45,3zM45,70h11v11h-11z" fill="#371">
+  </path>
+  <circle cx="50" cy="50" r="45" fill="none" stroke="#371" stroke-width="10">
+  </circle>
+
+  </svg>
+</body></html>
+EOX
+  end
+
+  def test_xlink
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>XLINK</title></head>
+<body>
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
+  <defs xmlns:l="http://www.w3.org/1999/xlink">
+    <radialGradient id="s1" fx=".4" fy=".2" r=".7">
+    <stop stop-color="#FE8"/>
+    <stop stop-color="#D70" offset="1"/>
+    </radialGradient>
+    <radialGradient id="s2" fx=".8" fy=".5" l:href="#s1"/>
+    <radialGradient id="s3" fx=".5" fy=".9" l:href="#s1"/>
+    <radialGradient id="s4" fx=".1" fy=".5" l:href="#s1"/>
+  </defs>
+  <g stroke="#940">
+    <path d="M73,29c-37-40-62-24-52,4l6-7c-8-16,7-26,42,9z" fill="url(#s1)"/>
+    <path d="M47,8c33-16,48,21,9,47l-6-5c38-27,20-44,5-37z" fill="url(#s2)"/>
+    <path d="M77,32c22,30,10,57-39,51l-1-8c3,3,67,5,36-36z" fill="url(#s3)"/>
+
+    <path d="M58,84c-4,20-38-4-8-24l-6-5c-36,43,15,56,23,27z" fill="url(#s4)"/>
+    <path d="M40,14c-40,37-37,52-9,68l1-8c-16-13-29-21,16-56z" fill="url(#s1)"/>
+    <path d="M31,33c19,23,20,7,35,41l-9,1.7c-4-19-8-14-31-37z" fill="url(#s2)"/>
+  </g>
+  </svg>
+</body></html>
+EOX
+  end
+
+  def test_br
+    assert_xhtml_equal <<EOX1
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>BR</title></head>
+<body>
+<br/>
+</body></html>
+EOX1
+  end
+
+  def test_strong
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>STRONG</title></head>
+<body>
+<strong></strong>
+</body></html>
+EOX
+  end
+
+  def test_script
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>SCRIPT</title></head>
+<body>
+<script>1 &lt; 2 &amp; 3</script>
+</body></html>
+EOX
+  end
+
+  def test_script_src
+    assert_xhtml_equal <<EOX1, <<EOX2.strip
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>SCRIPT</title><script src="http://example.com"/></head>
+<body>
+<script>1 &lt; 2 &amp; 3</script>
+</body></html>
+EOX1
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>SCRIPT</title><script src="http://example.com"></script></head>
+<body>
+<script>1 &lt; 2 &amp; 3</script>
+</body></html>
+EOX2
+  end
+
+  def test_title
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>1 &lt; 2 &amp; 3</title></head>
+<body>
+</body></html>
+EOX
+  end
+
+  def test_prolog
+    assert_xhtml_equal <<EOX1, <<EOX2.strip
+<?xml version="1.0" encoding="UTF-8" ?>
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>PROLOG</title></head>
+<body>
+</body></html>
+EOX1
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>PROLOG</title></head>
+<body>
+</body></html>
+EOX2
+  end
+
+end

data/tests/test_parser.rb

@@ -0,0 +1,68 @@
+require File.join(File.dirname(__FILE__), 'preamble')
+
+require 'html5/treebuilders'
+require 'html5/html5parser'
+
+
+$tree_types_to_test = ['simpletree', 'rexml']
+
+begin
+  require 'hpricot'
+  $tree_types_to_test.push('hpricot')
+rescue LoadError
+end
+
+$CHECK_PARSER_ERRORS = ARGV.delete('-p') # TODO
+
+puts 'Testing tree builders: ' + $tree_types_to_test * ', '
+
+
+class Html5ParserTestCase < Test::Unit::TestCase
+  include HTML5
+  include TestSupport
+
+  html5_test_files('tree-construction').each do |test_file|
+
+    test_name = File.basename(test_file).sub('.dat', '')
+
+    TestData.new(test_file, %w(data errors document-fragment document)).
+      each_with_index do |(input, errors, inner_html, expected), index|
+
+      errors = errors.split("\n")
+      expected = expected.gsub("\n| ","\n")[2..-1]
+
+      $tree_types_to_test.each do |tree_name|
+        define_method 'test_%s_%d_%s' % [ test_name, index + 1, tree_name ] do
+
+          parser = HTMLParser.new(:tree => TreeBuilders[tree_name])
+        
+          if inner_html
+            parser.parse_fragment(input, inner_html)
+          else
+            parser.parse(input)
+          end
+        
+          actual_output = convertTreeDump(parser.tree.testSerializer(parser.tree.document))
+
+          assert_equal sortattrs(expected), sortattrs(actual_output), [
+            '', 'Input:', input,
+            '', 'Expected:', expected,
+            '', 'Recieved:', actual_output
+          ].join("\n")
+
+          actual_errors = parser.errors.map do |(line, col), message|
+            'Line: %i Col: %i %s' % [line, col, message]
+          end
+          assert_equal errors.length, parser.errors.length, [
+            '', 'Input', input,
+            '', "Expected errors (#{errors.length}):", errors.join("\n"),
+            '', "Actual errors (#{actual_errors.length}):",
+                 actual_errors.join("\n")
+          ].join("\n")
+          
+        end
+      end
+    end
+  end
+
+end