html5 0.1.0 → 0.10.0

data/testdata/validator/style-scoped-attribute.test

@@ -0,0 +1,7 @@
+{"tests": [
+
+{"description": "invalid style scoped attribute value 'inherit'",
+"input": "<style scoped=inherit>",
+"fail-unless": "invalid-boolean-value"}
+
+]}

data/testdata/validator/tabindexattribute.test

@@ -0,0 +1,79 @@
+{"tests": [
+
+{"description": "valid tabindex attribute value '-1'",
+"input": "<span tabindex=-1>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value '0'",
+"input": "<span tabindex=0>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value '1'",
+"input": "<span tabindex=1>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value '32768'",
+"input": "<span tabindex=32768>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value '-32768'",
+"input": "<span tabindex=-32768>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value with leading spaces",
+"input": "<span tabindex=' -32768'>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value with trailing spaces",
+"input": "<span tabindex='-32768 '>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value with trailing junk",
+"input": "<span tabindex='32768a'>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value with trailing junk and whitespace",
+"input": "<span tabindex='32768a '>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value with trailing whitespace and junk",
+"input": "<span tabindex='32768 a'>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value with leading spaces",
+"input": "<span tabindex=' 32768'>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value with leading spaces (with sign)",
+"input": "<span tabindex=' -32768'>",
+"fail-if": "invalid-integer-value"},
+
+{"description": "invalid tabindex attribute value (blank)",
+"input": "<span tabindex>",
+"fail-unless": "attribute-value-can-not-be-blank"},
+
+{"description": "invalid tabindex attribute value due to leading junk",
+"input": "<span tabindex=a1>",
+"fail-unless": "invalid-integer-value"},
+
+{"description": "invalid tabindex attribute value due to two hyphens",
+"input": "<span tabindex=--1>",
+"fail-unless": "invalid-integer-value"},
+
+{"description": "invalid tabindex attribute value due to non-numeric",
+"input": "<span tabindex=foo>",
+"fail-unless": "invalid-integer-value"},
+
+{"description": "invalid tabindex attribute value due to positive sign",
+"input": "<span tabindex=+1>",
+"fail-unless": "invalid-integer-value"},
+
+{"description": "invalid tabindex attribute value due to decimal point",
+"input": "<span tabindex=.1>",
+"fail-unless": "invalid-integer-value"},
+
+{"description": "valid tabindex attribute value with trailing decimal point",
+"input": "<span tabindex=1.0>",
+"fail-if": "invalid-integer-value"}
+
+]}

data/tests/preamble.rb

@@ -8,28 +8,18 @@ else
   TESTDATA_DIR = File.join(File.dirname(File.dirname(File.expand_path(__FILE__))), 'testdata')
 end
 
-$:.unshift File.join(File.dirname(File.dirname(__FILE__)),'lib')
+# $:.unshift File.join(File.dirname(File.dirname(__FILE__)), 'lib')
 
-$:.unshift File.dirname(__FILE__)
+# $:.unshift File.dirname(__FILE__)
+
+require 'core_ext/string'
 
 def html5_test_files(subdirectory)
   Dir[File.join(TESTDATA_DIR, subdirectory, '*.*')]
 end
 
-begin
-  require 'rubygems'
-  require 'json'
-rescue LoadError
-  class JSON
-    def self.parse json
-      json.gsub!(/\$/, "\\$")
-      json.gsub!(/"\s*:/, '"=>')
-      json.gsub!(/\\u[0-9a-fA-F]{4}/) {|x| [x[2..-1].to_i(16)].pack('U')}
-      null = nil
-      eval json
-    end
-  end
-end
+require 'rubygems'
+require 'json'
 
 module HTML5
   module TestSupport
@@ -54,7 +44,7 @@ module HTML5
     
       def each
         data = {}
-        key=nil
+        key = nil
         @f.each_line do |line|
           if line[0] == ?# and @sections.include?(line[1..-2])
             heading = line[1..-2]

data/tests/test_encoding.rb

@@ -10,7 +10,7 @@ class Html5EncodingTestCase < Test::Unit::TestCase
     require 'rubygems'
     require 'UniversalDetector'
 
-    def test_chardet
+    def test_chardet #TODO: can we get rid of this?
       file = File.open(File.join(TESTDATA_DIR, 'encoding', 'chardet', 'test_big5.txt'), 'r')
       stream = HTML5::HTMLInputStream.new(file, :chardet => true)
       assert_equal 'big5', stream.char_encoding.downcase

data/tests/test_lxp.rb

@@ -257,6 +257,22 @@ EOX1
 <head><title>PROLOG</title></head>
 <body>
 </body></html>
+EOX2
+  end
+
+  def test_tagsoup
+    assert_xhtml_equal <<EOX1, <<EOX2.strip
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>TAGSOUP</title></head>
+<body>
+<u><blockquote><p></u>
+</body></html>
+EOX1
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>TAGSOUP</title></head>
+<body>
+<u/><blockquote><u/><p><u/>
+</p></blockquote></body></html>
 EOX2
   end
 

data/tests/test_parser.rb

@@ -50,8 +50,8 @@ class Html5ParserTestCase < Test::Unit::TestCase
             '', 'Recieved:', actual_output
           ].join("\n")
 
-          actual_errors = parser.errors.map do |(line, col), message|
-            'Line: %i Col: %i %s' % [line, col, message]
+          actual_errors = parser.errors.map do |(line, col), message, datavars|
+            'Line: %i Col: %i %s' % [line, col, E[message] % datavars]
           end
           assert_equal errors.length, parser.errors.length, [
             '', 'Input', input,

data/tests/test_sniffer.rb

@@ -0,0 +1,27 @@
+require File.join(File.dirname(__FILE__), 'preamble')
+require "html5/sniffer"
+
+class TestFeedTypeSniffer < Test::Unit::TestCase
+  include HTML5
+  include TestSupport
+  include Sniffer
+  
+  html5_test_files('sniffer').each do |test_file|
+    test_name = File.basename(test_file).sub('.test', '')
+
+    tests = JSON.parse(File.read(test_file))
+
+    tests.each_with_index do |data, index|
+      define_method('test_%s_%d' % [test_name, index + 1]) do
+        assert_equal data['type'], html_or_feed(data['input'])
+      end
+    end
+  end
+  # each_with_index do |t, i|
+  #     define_method "test_#{i}" do
+  #       assert_equal t[0], sniff_feed_type(t[1])
+  #     end
+  #   end
+  
+
+end

data/tests/test_treewalkers.rb

@@ -47,31 +47,29 @@ class TestTreeWalkers < Test::Unit::TestCase
     indent = 0
     concatenateCharacterTokens(tokens) do |token|
       case token[:type]
-        when :StartTag, :EmptyTag
-            output << "#{' '*indent}<#{token[:name]}>"
-            indent += 2
-            for name, value in token[:data].to_a.sort
-                next if name=='xmlns'
-                output << "#{' '*indent}#{name}=\"#{value}\""
-            end
-            indent -= 2 if token[:type] == :EmptyTag
-        when :EndTag
-            indent -= 2
-        when :Comment
-            output << "#{' '*indent}<!-- #{token[:data]} -->"
-        when :Doctype
-            if token[:name] and token[:name].any?
-              output << "#{' '*indent}<!DOCTYPE #{token[:name]}>"
-            else
-              output << "#{' '*indent}<!DOCTYPE >"
-            end
-        when :Characters, :SpaceCharacters
-            output << "#{' '*indent}\"#{token[:data]}\""
+      when :StartTag, :EmptyTag
+        output << "#{' '*indent}<#{token[:name]}>"
+        indent += 2
+        for name, value in token[:data].to_a.sort
+          next if name=='xmlns'
+          output << "#{' '*indent}#{name}=\"#{value}\""
+        end
+        indent -= 2 if token[:type] == :EmptyTag
+      when :EndTag
+        indent -= 2
+      when :Comment
+        output << "#{' '*indent}<!-- #{token[:data]} -->"
+      when :Doctype
+        if token[:name] and token[:name].any?
+          output << "#{' '*indent}<!DOCTYPE #{token[:name]}>"
         else
-            # TODO: what to do with errors?
+          output << "#{' '*indent}<!DOCTYPE >"
+        end
+      when :Characters, :SpaceCharacters
+        output << "#{' '*indent}\"#{token[:data]}\""
       end
     end
-    return output.join("\n")
+    output.join("\n")
   end
 
   html5_test_files('tree-construction').each do |test_file|
@@ -113,4 +111,25 @@ class TestTreeWalkers < Test::Unit::TestCase
       end
    end
   end
+
+  def test_all_tokens
+    expected = [
+        {:data => [], :type => :StartTag, :name => 'html'},
+        {:data => [], :type => :StartTag, :name => 'head'},
+        {:data => [], :type => :EndTag,   :name => 'head'},
+        {:data => [], :type => :StartTag, :name => 'body'},
+        {:data => [], :type => :EndTag,   :name => 'body'},
+        {:data => [], :type => :EndTag,   :name => 'html'}]
+    for treeName, tree_class in $tree_types_to_test
+      p = HTML5::HTMLParser.new(:tree => tree_class[:builder])
+      document = p.parse("<html></html>")
+      # document = tree_class.get(:adapter)(document)
+      output = tree_class[:walker].new(document)
+      expected.zip(output) do |expected_token, output_token|
+        assert_equal(expected_token, output_token)
+      end
+    end
+  end
+
+
 end

data/tests/test_validator.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby -wKU
+
+require File.join(File.dirname(__FILE__), 'preamble')
+
+require 'html5'
+require 'html5/filters/validator'
+
+class TestValidator < Test::Unit::TestCase
+  def run_validator_test(test)
+    p = HTML5::HTMLParser.new(:tokenizer => HTMLConformanceChecker)
+    p.parse(test['input'])
+    errorCodes = p.errors.collect{|e| e[1]}
+    if test.has_key?('fail-if')
+      assert !errorCodes.include?(test['fail-if'])
+    end
+    if test.has_key?('fail-unless')
+      assert errorCodes.include?(test['fail-unless'])
+    end
+  end
+
+  for filename in html5_test_files('validator')
+    tests    = JSON.load(open(filename))
+    testName = File.basename(filename).sub(".test", "")
+    tests['tests'].each_with_index do |test, index|
+      define_method "test_#{testName}_#{index}" do
+        run_validator_test(test)
+      end
+    end
+  end
+end
+

metadata

@@ -3,15 +3,15 @@ rubygems_version: 0.9.2
 specification_version: 1
 name: html5
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
-date: 2007-08-07 00:00:00 -07:00
+  version: 0.10.0
+date: 2007-10-08 00:00:00 -07:00
 summary: HTML5 parser/tokenizer.
 require_paths: 
 - lib
 email: ryan@theryanking.com
 homepage: http://code.google.com/p/html5lib
 rubyforge_project: html5
-description: A ruby based HTML parser/tokenizer based on the WHATWG HTML5 specification for maximum compatibility with major desktop web browsers.
+description: "A ruby implementation of the parsing algorithm in HTML5.  == FEATURES/PROBLEMS:  == SYNOPSIS:  TODO"
 autorequire: 
 default_executable: 
 bindir: bin
@@ -33,12 +33,18 @@ files:
 - Manifest.txt
 - README
 - Rakefile.rb
+- bin/html5
+- lib/core_ext/string.rb
 - lib/html5.rb
 - lib/html5/constants.rb
 - lib/html5/filters/base.rb
 - lib/html5/filters/inject_meta_charset.rb
+- lib/html5/filters/iso639codes.rb
 - lib/html5/filters/optionaltags.rb
+- lib/html5/filters/rfc2046.rb
+- lib/html5/filters/rfc3987.rb
 - lib/html5/filters/sanitizer.rb
+- lib/html5/filters/validator.rb
 - lib/html5/filters/whitespace.rb
 - lib/html5/html5parser.rb
 - lib/html5/html5parser/after_body_phase.rb
@@ -65,6 +71,7 @@ files:
 - lib/html5/serializer.rb
 - lib/html5/serializer/htmlserializer.rb
 - lib/html5/serializer/xhtmlserializer.rb
+- lib/html5/sniffer.rb
 - lib/html5/tokenizer.rb
 - lib/html5/treebuilders.rb
 - lib/html5/treebuilders/base.rb
@@ -76,16 +83,68 @@ files:
 - lib/html5/treewalkers/hpricot.rb
 - lib/html5/treewalkers/rexml.rb
 - lib/html5/treewalkers/simpletree.rb
-- parse.rb
+- lib/html5/version.rb
+- testdata/encoding/chardet/test_big5.txt
+- testdata/encoding/test-yahoo-jp.dat
+- testdata/encoding/tests1.dat
+- testdata/encoding/tests2.dat
+- testdata/sanitizer/tests1.dat
+- testdata/serializer/core.test
+- testdata/serializer/injectmeta.test
+- testdata/serializer/optionaltags.test
+- testdata/serializer/options.test
+- testdata/serializer/whitespace.test
+- testdata/sites/google-results.htm
+- testdata/sites/python-ref-import.htm
+- testdata/sites/web-apps-old.htm
+- testdata/sites/web-apps.htm
+- testdata/sniffer/htmlOrFeed.json
+- testdata/tokenizer/contentModelFlags.test
+- testdata/tokenizer/entities.test
+- testdata/tokenizer/escapeFlag.test
+- testdata/tokenizer/test1.test
+- testdata/tokenizer/test2.test
+- testdata/tokenizer/test3.test
+- testdata/tokenizer/test4.test
+- testdata/tree-construction/tests1.dat
+- testdata/tree-construction/tests2.dat
+- testdata/tree-construction/tests3.dat
+- testdata/tree-construction/tests4.dat
+- testdata/tree-construction/tests5.dat
+- testdata/tree-construction/tests6.dat
+- testdata/validator/attributes.test
+- testdata/validator/base-href-attribute.test
+- testdata/validator/base-target-attribute.test
+- testdata/validator/blockquote-cite-attribute.test
+- testdata/validator/classattribute.test
+- testdata/validator/contenteditableattribute.test
+- testdata/validator/contextmenuattribute.test
+- testdata/validator/dirattribute.test
+- testdata/validator/draggableattribute.test
+- testdata/validator/html-xmlns-attribute.test
+- testdata/validator/idattribute.test
+- testdata/validator/inputattributes.test
+- testdata/validator/irrelevantattribute.test
+- testdata/validator/langattribute.test
+- testdata/validator/li-value-attribute.test
+- testdata/validator/link-href-attribute.test
+- testdata/validator/link-hreflang-attribute.test
+- testdata/validator/link-rel-attribute.test
+- testdata/validator/ol-start-attribute.test
+- testdata/validator/starttags.test
+- testdata/validator/style-scoped-attribute.test
+- testdata/validator/tabindexattribute.test
 - tests/preamble.rb
 - tests/test_encoding.rb
 - tests/test_lxp.rb
 - tests/test_parser.rb
 - tests/test_sanitizer.rb
 - tests/test_serializer.rb
+- tests/test_sniffer.rb
 - tests/test_stream.rb
 - tests/test_tokenizer.rb
 - tests/test_treewalkers.rb
+- tests/test_validator.rb
 - tests/tokenizer_test_parser.rb
 test_files: []
 
@@ -93,8 +152,8 @@ rdoc_options: []
 
 extra_rdoc_files: []
 
-executables: []
-
+executables: 
+- html5
 extensions: []
 
 requirements: []