spk-html5 0.10.1

data/test/test_treewalkers.rb

@@ -0,0 +1,135 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'preamble'))
+
+require 'html5/html5parser'
+require 'html5/treewalkers'
+require 'html5/treebuilders'
+
+$tree_types_to_test = {
+  'simpletree' =>
+    {:builder => HTML5::TreeBuilders['simpletree'],
+     :walker  => HTML5::TreeWalkers['simpletree']},
+  'rexml' =>
+    {:builder => HTML5::TreeBuilders['rexml'],
+     :walker  => HTML5::TreeWalkers['rexml']},
+  'hpricot' =>
+    {:builder => HTML5::TreeBuilders['hpricot'],
+     :walker  => HTML5::TreeWalkers['hpricot']},
+}
+
+puts 'Testing tree walkers: ' + $tree_types_to_test.keys * ', '
+
+class TestTreeWalkers < Test::Unit::TestCase
+  include HTML5::TestSupport
+
+  def concatenateCharacterTokens(tokens)
+    charactersToken = nil
+    for token in tokens
+        type = token[:type]
+        if [:Characters, :SpaceCharacters].include?(type)
+            if charactersToken == nil
+                charactersToken = {:type => :Characters, :data => token[:data]}
+            else
+                charactersToken[:data] += token[:data]
+            end
+        else
+            if charactersToken != nil
+                yield charactersToken
+                charactersToken = nil
+            end
+            yield token
+        end
+    end
+    yield charactersToken if charactersToken != nil
+  end
+
+  def convertTokens(tokens)
+    output = []
+    indent = 0
+    concatenateCharacterTokens(tokens) do |token|
+      case token[:type]
+      when :StartTag, :EmptyTag
+        output << "#{' '*indent}<#{token[:name]}>"
+        indent += 2
+        for name, value in token[:data].to_a.sort
+          next if name=='xmlns'
+          output << "#{' '*indent}#{name}=\"#{value}\""
+        end
+        indent -= 2 if token[:type] == :EmptyTag
+      when :EndTag
+        indent -= 2
+      when :Comment
+        output << "#{' '*indent}<!-- #{token[:data]} -->"
+      when :Doctype
+        if token[:name] and token[:name].any?
+          output << "#{' '*indent}<!DOCTYPE #{token[:name]}>"
+        else
+          output << "#{' '*indent}<!DOCTYPE >"
+        end
+      when :Characters, :SpaceCharacters
+        output << "#{' '*indent}\"#{token[:data]}\""
+      end
+    end
+    output.join("\n")
+  end
+
+  html5_test_files('tree-construction').each do |test_file|
+
+    test_name = File.basename(test_file).sub('.dat', '')
+    next if test_name == 'tests5' # TODO
+
+    TestData.new(test_file, %w(data errors document-fragment document)).
+      each_with_index do |(input, errors, inner_html, expected), index|
+
+      expected = expected.gsub("\n| ","\n")[2..-1]
+
+      $tree_types_to_test.each do |tree_name, tree_class|
+
+        define_method "test_#{test_name}_#{index}_#{tree_name}" do
+
+          parser = HTML5::HTMLParser.new(:tree => tree_class[:builder])
+
+          if inner_html
+            parser.parse_fragment(input, inner_html)
+          else
+            parser.parse(input)
+          end
+
+          document = parser.tree.get_document
+
+          begin
+            output = sortattrs(convertTokens(tree_class[:walker].new(document)))
+            expected = sortattrs(expected)
+            assert_equal expected, output, [
+              '', 'Input:', input,
+              '', 'Expected:', expected,
+              '', 'Received:', output
+            ].join("\n")
+          rescue NotImplementedError
+            # Amnesty for those that confess...
+          end
+        end
+      end
+   end
+  end
+
+  def test_all_tokens
+    expected = [
+        {:data => [], :type => :StartTag, :name => 'html'},
+        {:data => [], :type => :StartTag, :name => 'head'},
+        {:data => [], :type => :EndTag,   :name => 'head'},
+        {:data => [], :type => :StartTag, :name => 'body'},
+        {:data => [], :type => :EndTag,   :name => 'body'},
+        {:data => [], :type => :EndTag,   :name => 'html'}]
+    for treeName, tree_class in $tree_types_to_test
+      p = HTML5::HTMLParser.new(:tree => tree_class[:builder])
+      document = p.parse("<html></html>")
+      # document = tree_class.get(:adapter)(document)
+      output = tree_class[:walker].new(document)
+      expected.zip(output) do |expected_token, output_token|
+        assert_equal(expected_token, output_token)
+      end
+    end
+  end
+
+
+end

data/test/test_validator.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby -wKU
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'preamble'))
+
+require 'html5'
+require 'html5/filters/validator'
+
+class TestValidator < Test::Unit::TestCase
+  def run_validator_test(test)
+    p = HTML5::HTMLParser.new(:tokenizer => HTMLConformanceChecker)
+    p.parse(test['input'])
+    errorCodes = p.errors.collect{|e| e[1]}
+    if test.has_key?('fail-if')
+      assert !errorCodes.include?(test['fail-if'])
+    end
+    if test.has_key?('fail-unless')
+      assert errorCodes.include?(test['fail-unless'])
+    end
+  end
+
+  for filename in html5_test_files('validator')
+    tests    = JSON.load(open(filename))
+    testName = File.basename(filename).sub(".test", "")
+    tests['tests'].each_with_index do |test, index|
+      define_method "test_#{testName}_#{index}" do
+        run_validator_test(test)
+      end
+    end
+  end
+end
+

data/test/tokenizer_test_parser.rb

@@ -0,0 +1,67 @@
+require 'html5/constants'
+
+class TokenizerTestParser
+  def initialize(tokenizer)
+    @tokenizer = tokenizer
+  end
+
+  def parse
+    @outputTokens = []
+
+    debug = nil
+    for token in @tokenizer
+      debug = token.inspect if token[:type] == :ParseError
+      send(('process' + token[:type].to_s), token)
+    end
+
+    return @outputTokens
+  end
+
+  def processDoctype(token)
+    @outputTokens.push(["DOCTYPE", token[:name], token[:publicId],
+      token[:systemId], token[:correct]])
+  end
+
+  def processStartTag(token)
+    if token[:self_closing]
+      @outputTokens.push(["StartTag", token[:name], token[:data], token[:self_closing]])
+    else
+      @outputTokens.push(["StartTag", token[:name], token[:data]])
+    end
+  end
+
+  def processEmptyTag(token)
+    if not HTML5::VOID_ELEMENTS.include? token[:name]
+      @outputTokens.push("ParseError")
+    end
+    @outputTokens.push(["StartTag", token[:name], token[:data]])
+  end
+
+  def processEndTag(token)
+    if token[:data].length > 0
+      self.processParseError(token)
+    end
+    @outputTokens.push(["EndTag", token[:name]])
+  end
+
+  def processComment(token)
+    @outputTokens.push(["Comment", token[:data]])
+  end
+
+  def processCharacters(token)
+    @outputTokens.push(["Character", token[:data]])
+  end
+
+  alias processSpaceCharacters processCharacters
+
+  def processCharacters(token)
+    @outputTokens.push(["Character", token[:data]])
+  end
+
+  def process_eof(token)
+  end
+
+  def processParseError(token)
+    @outputTokens.push("ParseError")
+  end
+end

data/test19.rb

@@ -0,0 +1,38 @@
+# 
+# This temporary test driver tracks progress on getting HTML5lib working
+# on Ruby 1.9.  Prereqs of Hoe, Hpricot, and UniversalDetector will be
+# required to complete this.
+#
+# Once all the tests pass, this file should be deleted
+#
+
+require 'test/test_cli'
+
+# requires UniversalDetector
+# require 'test/test_encoding'
+
+require 'test/test_input_stream'
+
+require 'test/test_lxp'
+
+require 'test/test_parser'
+
+# warning: method redefined; discarding old test
+# warning: instance variable @expanded_name not initialized
+# SimpleDelegator.class
+# require 'test/test_sanitizer'
+
+require 'test/test_serializer'
+
+require 'test/test_sniffer'
+
+require 'test/test_stream'
+
+# warning: shadowing outer local variable - tokens
+# require 'test/test_tokenizer'
+
+# requires hpricot
+# require 'test/test_treewalkers'
+
+# warning: instance variable @delegate_sd_obj not initialized
+# require 'test/test_validator'

metadata

@@ -0,0 +1,198 @@
+--- !ruby/object:Gem::Specification 
+name: spk-html5
+version: !ruby/object:Gem::Version 
+  hash: 53
+  prerelease: false
+  segments: 
+  - 0
+  - 10
+  - 1
+  version: 0.10.1
+platform: ruby
+authors: 
+- Ryan King
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-05 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rchardet
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 1
+        - 3
+        version: "1.3"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rubyforge
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 2
+        - 0
+        - 4
+        version: 2.0.4
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 19
+        segments: 
+        - 2
+        - 6
+        - 2
+        version: 2.6.2
+  type: :development
+  version_requirements: *id003
+description: A ruby implementation of the parsing algorithm in HTML5.
+email: ryan@theryanking.com
+executables: 
+- html5
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+files: 
+- History.txt
+- Manifest.txt
+- README
+- Rakefile.rb
+- bin/html5
+- lib/html5.rb
+- lib/html5/cli.rb
+- lib/html5/constants.rb
+- lib/html5/filters/base.rb
+- lib/html5/filters/inject_meta_charset.rb
+- lib/html5/filters/iso639codes.rb
+- lib/html5/filters/optionaltags.rb
+- lib/html5/filters/rfc2046.rb
+- lib/html5/filters/rfc3987.rb
+- lib/html5/filters/sanitizer.rb
+- lib/html5/filters/validator.rb
+- lib/html5/filters/whitespace.rb
+- lib/html5/html5parser.rb
+- lib/html5/html5parser/after_after_body_phase.rb
+- lib/html5/html5parser/after_after_frameset_phase.rb
+- lib/html5/html5parser/after_body_phase.rb
+- lib/html5/html5parser/after_frameset_phase.rb
+- lib/html5/html5parser/after_head_phase.rb
+- lib/html5/html5parser/before_head_phase.rb
+- lib/html5/html5parser/before_html_phase.rb
+- lib/html5/html5parser/in_body_phase.rb
+- lib/html5/html5parser/in_caption_phase.rb
+- lib/html5/html5parser/in_cell_phase.rb
+- lib/html5/html5parser/in_column_group_phase.rb
+- lib/html5/html5parser/in_foreign_content_phase.rb
+- lib/html5/html5parser/in_frameset_phase.rb
+- lib/html5/html5parser/in_head_phase.rb
+- lib/html5/html5parser/in_row_phase.rb
+- lib/html5/html5parser/in_select_phase.rb
+- lib/html5/html5parser/in_select_table_phase.rb
+- lib/html5/html5parser/in_table_body_phase.rb
+- lib/html5/html5parser/in_table_phase.rb
+- lib/html5/html5parser/initial_phase.rb
+- lib/html5/html5parser/phase.rb
+- lib/html5/inputstream.rb
+- lib/html5/liberalxmlparser.rb
+- lib/html5/sanitizer.rb
+- lib/html5/serializer.rb
+- lib/html5/serializer/htmlserializer.rb
+- lib/html5/serializer/xhtmlserializer.rb
+- lib/html5/sniffer.rb
+- lib/html5/tokenizer.rb
+- lib/html5/treebuilders.rb
+- lib/html5/treebuilders/base.rb
+- lib/html5/treebuilders/hpricot.rb
+- lib/html5/treebuilders/rexml.rb
+- lib/html5/treebuilders/simpletree.rb
+- lib/html5/treewalkers.rb
+- lib/html5/treewalkers/base.rb
+- lib/html5/treewalkers/hpricot.rb
+- lib/html5/treewalkers/rexml.rb
+- lib/html5/treewalkers/simpletree.rb
+- lib/html5/version.rb
+- test/preamble.rb
+- test/test_cli.rb
+- test/test_encoding.rb
+- test/test_input_stream.rb
+- test/test_lxp.rb
+- test/test_parser.rb
+- test/test_sanitizer.rb
+- test/test_serializer.rb
+- test/test_sniffer.rb
+- test/test_stream.rb
+- test/test_tokenizer.rb
+- test/test_treewalkers.rb
+- test/test_validator.rb
+- test/tokenizer_test_parser.rb
+- test19.rb
+has_rdoc: true
+homepage: http://code.google.com/p/html5lib
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: spk-html5
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: HTML5 parser/tokenizer.
+test_files: 
+- test/test_sniffer.rb
+- test/test_treewalkers.rb
+- test/test_input_stream.rb
+- test/test_stream.rb
+- test/test_encoding.rb
+- test/test_serializer.rb
+- test/test_validator.rb
+- test/test_tokenizer.rb
+- test/test_sanitizer.rb
+- test/test_parser.rb
+- test/test_cli.rb
+- test/test_lxp.rb