spk-html5 0.10.1

data/lib/html5/treewalkers/hpricot.rb

@@ -0,0 +1,48 @@
+require 'html5/treewalkers/base'
+require 'rexml/document'
+
+module HTML5
+  module TreeWalkers
+    module Hpricot
+      class TreeWalker < HTML5::TreeWalkers::NonRecursiveTreeWalker
+
+        def node_details(node)
+          case node
+          when ::Hpricot::Elem
+            if node.name.empty?
+              [:DOCUMENT_FRAGMENT]
+            else
+              [:ELEMENT, node.name,
+                node.attributes.map {|name, value| [name, value]},
+                !node.empty?]
+            end
+          when ::Hpricot::Text
+            [:TEXT, node.content]
+          when ::Hpricot::Comment
+            [:COMMENT, node.content]
+          when ::Hpricot::Doc
+            [:DOCUMENT]
+          when ::Hpricot::DocType
+            [:DOCTYPE, node.target, node.public_id, node.system_id]
+          when ::Hpricot::XMLDecl
+            [nil]
+          else
+            [:UNKNOWN, node.class.inspect]
+          end
+        end
+
+        def first_child(node)
+          node.children.first
+        end
+
+        def next_sibling(node)
+          node.next_node
+        end
+
+        def parent(node)
+          node.parent
+        end
+      end
+    end
+  end
+end

data/lib/html5/treewalkers/rexml.rb

@@ -0,0 +1,48 @@
+require 'html5/treewalkers/base'
+require 'rexml/document'
+
+module HTML5
+  module TreeWalkers
+    module REXML
+      class TreeWalker < HTML5::TreeWalkers::NonRecursiveTreeWalker
+
+        def node_details(node)
+          case node
+          when ::REXML::Document
+            [:DOCUMENT]
+          when ::REXML::Element
+            if !node.name
+              [:DOCUMENT_FRAGMENT]
+            else
+              [:ELEMENT, node.name,
+                node.attributes.map {|name,value| [name,value]},
+                node.has_elements? || node.has_text?]
+            end
+          when ::REXML::Text
+            [:TEXT, node.value]
+          when ::REXML::Comment
+            [:COMMENT, node.string]
+          when ::REXML::DocType
+            [:DOCTYPE, node.name, node.public, node.system]
+          when ::REXML::XMLDecl
+            [nil]
+          else
+            [:UNKNOWN, node.class.inspect]
+          end
+        end
+
+        def first_child(node)
+          node.children.first
+        end
+
+        def next_sibling(node)
+          node.next_sibling
+        end
+
+        def parent(node)
+          node.parent
+        end
+      end
+    end
+  end
+end

data/lib/html5/treewalkers/simpletree.rb

@@ -0,0 +1,48 @@
+require 'html5/treewalkers/base'
+
+module HTML5
+  module TreeWalkers
+    module SimpleTree
+      class TreeWalker < HTML5::TreeWalkers::Base
+        include HTML5::TreeBuilders::SimpleTree
+
+        def walk(node)
+          case node
+          when Document, DocumentFragment
+            return
+
+          when DocumentType
+            yield doctype(node.name, node.public_id, node.system_id)
+
+          when TextNode
+            text(node.value) {|token| yield token}
+
+          when Element
+            if VOID_ELEMENTS.include?(node.name)
+              yield empty_tag(node.name, node.attributes, node.hasContent())
+            else
+              yield start_tag(node.name, node.attributes)
+              for child in node.childNodes
+                walk(child) {|token| yield token}
+              end
+              yield end_tag(node.name)
+            end
+
+          when CommentNode
+            yield comment(node.value)
+
+          else
+            puts '?'
+            yield unknown(node.class)
+          end
+        end
+
+        def each
+          for child in @tree.childNodes
+            walk(child) {|node| yield node}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html5/version.rb

@@ -0,0 +1,3 @@
+module HTML5
+  VERSION = '0.10.1'
+end

data/test/preamble.rb

@@ -0,0 +1,69 @@
+require 'test/unit'
+
+HTML5_BASE = File.dirname(File.dirname(File.dirname(File.expand_path(__FILE__)))) 
+
+if File.exists?(File.join(HTML5_BASE, 'ruby', 'testdata'))
+  TESTDATA_DIR = File.join(HTML5_BASE, 'ruby', 'testdata')
+else
+  TESTDATA_DIR = File.join(HTML5_BASE, 'testdata')
+end
+
+$:.unshift File.join(File.dirname(File.dirname(__FILE__)), 'lib')
+$:.unshift File.dirname(__FILE__)
+
+def html5_test_files(subdirectory)
+  Dir[File.join(TESTDATA_DIR, subdirectory, '*.*')]
+end
+
+require 'rubygems'
+require 'json'
+
+module HTML5
+  module TestSupport
+    # convert the output of str(document) to the format used in the testcases
+    def convertTreeDump(treedump)
+      treedump.split(/\n/)[1..-1].map { |line| (line.length > 2 and line[0] == ?|) ? line[3..-1] : line }.join("\n")
+    end
+
+    def sortattrs(output)
+      output.gsub(/^(\s+)\w+=.*(\n\1\w+=.*)+/) do |match|
+         match.split("\n").sort.join("\n")
+      end
+    end
+
+    class TestData
+      include Enumerable
+
+      def initialize(filename, sections)
+        @f = open(filename)
+        @sections = sections
+      end
+    
+      def each
+        data = {}
+        key = nil
+        @f.each_line do |line|
+          if line[0] == ?# and @sections.include?(line[1..-2])
+            heading = line[1..-2]
+            if data.any? and heading == @sections[0]
+              data[key].chomp!  #Remove trailing newline
+              yield normaliseOutput(data)
+              data = {}
+            end
+            key = heading
+            data[key]=""
+          elsif key
+            data[key] += line
+          end
+        end
+        yield normaliseOutput(data) if data
+      end
+        
+      def normaliseOutput(data)
+        #Remove trailing newlines
+        data.keys.each { |key| data[key].chomp! }
+        @sections.map {|heading| data[heading]}
+      end
+    end
+  end
+end

data/test/test_cli.rb

@@ -0,0 +1,16 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'preamble'))
+require "html5/cli"
+
+class TestCli < Test::Unit::TestCase
+  def test_open_input
+    assert_equal $stdin, HTML5::CLI.open_input('-')
+    assert_kind_of StringIO, HTML5::CLI.open_input('http://whatwg.org/')
+    assert_kind_of File, HTML5::CLI.open_input('testdata/sites/google-results.htm')
+  end
+  
+  def test_parse_opts
+    HTML5::CLI.parse_opts [] # TODO test defaults
+    assert_equal 'hpricot', HTML5::CLI.parse_opts(['-b', 'hpricot']).treebuilder
+    assert_equal 'hpricot', HTML5::CLI.parse_opts(['--treebuilder', 'hpricot']).treebuilder
+  end
+end

data/test/test_encoding.rb

@@ -0,0 +1,35 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'preamble'))
+
+require 'html5/inputstream'
+
+class Html5EncodingTestCase < Test::Unit::TestCase
+  include HTML5
+  include TestSupport
+
+  begin
+    require 'rubygems'
+    require 'UniversalDetector'
+
+    def test_chardet #TODO: can we get rid of this?
+      file = File.open(File.join(TESTDATA_DIR, 'encoding', 'chardet', 'test_big5.txt'), 'r')
+      stream = HTML5::HTMLInputStream.new(file, :chardet => true)
+      assert_equal 'big5', stream.char_encoding.downcase
+    rescue LoadError
+      puts "chardet not found, skipping chardet tests"
+    end
+  end
+
+  html5_test_files('encoding').each do |test_file|        
+    test_name = File.basename(test_file).sub('.dat', '').tr('-', '')
+
+    TestData.new(test_file, %w(data encoding)).
+      each_with_index do |(input, encoding), index|
+
+      define_method 'test_%s_%d' % [ test_name, index + 1 ] do
+        stream = HTML5::HTMLInputStream.new(input, :chardet => false)
+        assert_equal encoding.downcase, stream.char_encoding.downcase, input
+      end
+    end
+  end
+
+end

data/test/test_input_stream.rb

@@ -0,0 +1,26 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'preamble'))
+require "test/unit"
+require "html5/inputstream"
+
+class TestHtml5Inputstream < Test::Unit::TestCase
+  def test_newline_in_queue
+    stream = HTML5::HTMLInputStream.new("\nfoo")
+    stream.unget(stream.char)
+    assert_equal [1, 0], stream.position
+  end
+  
+  def test_buffer_boundary
+    stream = HTML5::HTMLInputStream.new("abcdefghijklmnopqrstuvwxyz" * 50, :encoding => 'windows-1252')
+    1022.times{stream.char}
+    assert_equal "i", stream.char
+  end
+  
+  def test_chars_until
+    stream = HTML5::HTMLInputStream.new("aaaaaaab")
+    assert_equal "aaaaaaa", stream.chars_until("b")
+
+    stream = HTML5::HTMLInputStream.new("aaaaaaab")
+    assert_equal "aaaaaaab", stream.chars_until("c")
+    
+  end
+end

data/test/test_lxp.rb

@@ -0,0 +1,283 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'preamble'))
+
+require 'html5/liberalxmlparser'
+
+XMLELEM = /<(\w+\s*)((?:[-:\w]+="[^"]*"\s*)+)(\/?)>/
+
+def assert_xml_equal(input, expected=nil, parser=HTML5::XMLParser)
+  sortattrs = proc {"<#{$1+$2.split.sort.join(' ')+$3}>"}
+  document = parser.parse(input.chomp, :lowercase_attr_name => false, :lowercase_element_name => false).root
+  if not expected
+    expected = input.chomp.gsub(XMLELEM,&sortattrs)
+    if expected.respond_to? :force_encoding
+      expected = expected.gsub(/&#(\d+);/) {$1.to_i.chr('utf-8')}
+    else
+      expected = expected.gsub(/&#(\d+);/) {[$1.to_i].pack('U')}
+    end
+    output = document.to_s.gsub(/'/,'"').gsub(XMLELEM,&sortattrs)
+    assert_equal(expected, output)
+  else
+    assert_equal(expected, document.to_s.gsub(/'/,'"'))
+  end
+end
+
+def assert_xhtml_equal(input, expected=nil, parser=HTML5::XHTMLParser)
+  assert_xml_equal(input, expected, parser)
+end
+
+class BasicXhtml5Test < Test::Unit::TestCase
+
+  def test_title_body_mismatched_close
+    assert_xhtml_equal(
+      '<title>Xhtml</title><b><i>content</b></i>',
+      '<html xmlns="http://www.w3.org/1999/xhtml">' +
+      '<head><title>Xhtml</title></head>' + 
+      '<body><b><i>content</i></b></body>' +
+      '</html>')
+  end
+
+  def test_title_body_named_charref
+    assert_xhtml_equal(
+      '<title>ntilde</title>A &ntilde B',
+      '<html xmlns="http://www.w3.org/1999/xhtml">' +
+      '<head><title>ntilde</title></head>' + 
+      '<body>A '+ [0xF1].pack('U') + ' B</body>' +
+      '</html>')
+  end
+end
+
+class BasicXmlTest < Test::Unit::TestCase
+
+  def test_comment
+    assert_xml_equal("<x><!-- foo --></x>")
+  end
+
+  def test_cdata
+    assert_xml_equal("<x><![CDATA[foo]]></x>","<x>foo</x>")
+  end
+
+  def test_simple_text
+    assert_xml_equal("<p>foo</p>","<p>foo</p>")
+  end
+
+  def test_optional_close
+    assert_xml_equal("<p>foo","<p>foo</p>")
+  end
+
+  def test_html_mismatched
+    assert_xml_equal("<b><i>foo</b></i>","<b><i>foo</i></b>")
+  end
+end
+
+class OpmlTest < Test::Unit::TestCase
+
+  def test_mixedCaseElement
+    assert_xml_equal(
+      '<opml version="1.0">' +
+      '<head><ownerName>Dave Winer</ownerName></head>' +
+      '</opml>')
+  end
+
+  def test_mixedCaseAttribute
+    assert_xml_equal(
+      '<opml version="1.0">' +
+      '<body><outline isComment="true"/></body>' +
+      '</opml>')
+  end
+
+  def test_malformed
+    assert_xml_equal(
+      '<opml version="1.0">' +
+      '<body><outline text="Odds & Ends"/></body>' +
+      '</opml>',
+      '<opml version="1.0">' +
+      '<body><outline text="Odds &amp; Ends"/></body>' +
+      '</opml>')
+  end
+end
+
+class XhtmlTest < Test::Unit::TestCase
+
+  def test_mathml
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>MathML</title></head>
+<body>
+  <math xmlns="http://www.w3.org/1998/Math/MathML">
+  <mrow>
+    <mi>x</mi>
+    <mo>=</mo>
+
+    <mfrac>
+    <mrow>
+      <mrow>
+      <mo>-</mo>
+      <mi>b</mi>
+      </mrow>
+      <mo>&#177;</mo>
+      <msqrt>
+
+      <mrow>
+        <msup>
+        <mi>b</mi>
+        <mn>2</mn>
+        </msup>
+        <mo>-</mo>
+        <mrow>
+
+        <mn>4</mn>
+        <mo>&#8290;</mo>
+        <mi>a</mi>
+        <mo>&#8290;</mo>
+        <mi>c</mi>
+        </mrow>
+      </mrow>
+
+      </msqrt>
+    </mrow>
+    <mrow>
+      <mn>2</mn>
+      <mo>&#8290;</mo>
+      <mi>a</mi>
+    </mrow>
+    </mfrac>
+
+  </mrow>
+  </math>
+</body></html>
+EOX
+  end
+
+  def test_svg
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>SVG</title></head>
+<body>
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
+  <path d="M38,38c0-12,24-15,23-2c0,9-16,13-16,23v7h11v-4c0-9,17-12,17-27
+       c-2-22-45-22-45,3zM45,70h11v11h-11z" fill="#371">
+  </path>
+  <circle cx="50" cy="50" r="45" fill="none" stroke="#371" stroke-width="10">
+  </circle>
+
+  </svg>
+</body></html>
+EOX
+  end
+
+  def test_xlink
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>XLINK</title></head>
+<body>
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
+  <defs xmlns:l="http://www.w3.org/1999/xlink">
+    <radialGradient id="s1" fx=".4" fy=".2" r=".7">
+    <stop stop-color="#FE8"/>
+    <stop stop-color="#D70" offset="1"/>
+    </radialGradient>
+    <radialGradient id="s2" fx=".8" fy=".5" l:href="#s1"/>
+    <radialGradient id="s3" fx=".5" fy=".9" l:href="#s1"/>
+    <radialGradient id="s4" fx=".1" fy=".5" l:href="#s1"/>
+  </defs>
+  <g stroke="#940">
+    <path d="M73,29c-37-40-62-24-52,4l6-7c-8-16,7-26,42,9z" fill="url(#s1)"/>
+    <path d="M47,8c33-16,48,21,9,47l-6-5c38-27,20-44,5-37z" fill="url(#s2)"/>
+    <path d="M77,32c22,30,10,57-39,51l-1-8c3,3,67,5,36-36z" fill="url(#s3)"/>
+
+    <path d="M58,84c-4,20-38-4-8-24l-6-5c-36,43,15,56,23,27z" fill="url(#s4)"/>
+    <path d="M40,14c-40,37-37,52-9,68l1-8c-16-13-29-21,16-56z" fill="url(#s1)"/>
+    <path d="M31,33c19,23,20,7,35,41l-9,1.7c-4-19-8-14-31-37z" fill="url(#s2)"/>
+  </g>
+  </svg>
+</body></html>
+EOX
+  end
+
+  def test_br
+    assert_xhtml_equal <<EOX1
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>BR</title></head>
+<body>
+<br/>
+</body></html>
+EOX1
+  end
+
+  def test_strong
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>STRONG</title></head>
+<body>
+<strong></strong>
+</body></html>
+EOX
+  end
+
+  def test_script
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>SCRIPT</title></head>
+<body>
+<script>1 &lt; 2 &amp; 3</script>
+</body></html>
+EOX
+  end
+
+  def test_script_src
+    assert_xhtml_equal <<EOX1, <<EOX2.strip
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>SCRIPT</title><script src="http://example.com"/></head>
+<body>
+<script>1 &lt; 2 &amp; 3</script>
+</body></html>
+EOX1
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>SCRIPT</title><script src="http://example.com"></script></head>
+<body>
+<script>1 &lt; 2 &amp; 3</script>
+</body></html>
+EOX2
+  end
+
+  def test_title
+    assert_xhtml_equal <<EOX
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>1 &lt; 2 &amp; 3</title></head>
+<body>
+</body></html>
+EOX
+  end
+
+  def test_prolog
+    assert_xhtml_equal <<EOX1, <<EOX2.strip
+<?xml version="1.0" encoding="UTF-8" ?>
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>PROLOG</title></head>
+<body>
+</body></html>
+EOX1
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>PROLOG</title></head>
+<body>
+</body></html>
+EOX2
+  end
+
+  def test_tagsoup
+    assert_xhtml_equal <<EOX1, <<EOX2.strip
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>TAGSOUP</title></head>
+<body>
+<u><blockquote><p></u>
+</body></html>
+EOX1
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>TAGSOUP</title></head>
+<body>
+<u/><blockquote><u/><p><u/>
+</p></blockquote></body></html>
+EOX2
+  end
+
+end