RubyGems - superfeedr-nokogiri - Versions diffs - 1.4.0.20091116183308 - Mend

superfeedr-nokogiri 1.4.0.20091116183308

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (270) hide show

data/.autotest +27 -0
data/CHANGELOG.ja.rdoc +330 -0
data/CHANGELOG.rdoc +314 -0
data/Manifest.txt +269 -0
data/README.ja.rdoc +105 -0
data/README.rdoc +118 -0
data/Rakefile +244 -0
data/bin/nokogiri +49 -0
data/ext/nokogiri/extconf.rb +145 -0
data/ext/nokogiri/html_document.c +145 -0
data/ext/nokogiri/html_document.h +10 -0
data/ext/nokogiri/html_element_description.c +272 -0
data/ext/nokogiri/html_element_description.h +10 -0
data/ext/nokogiri/html_entity_lookup.c +32 -0
data/ext/nokogiri/html_entity_lookup.h +8 -0
data/ext/nokogiri/html_sax_parser_context.c +92 -0
data/ext/nokogiri/html_sax_parser_context.h +11 -0
data/ext/nokogiri/nokogiri.c +89 -0
data/ext/nokogiri/nokogiri.h +145 -0
data/ext/nokogiri/xml_attr.c +92 -0
data/ext/nokogiri/xml_attr.h +9 -0
data/ext/nokogiri/xml_attribute_decl.c +67 -0
data/ext/nokogiri/xml_attribute_decl.h +9 -0
data/ext/nokogiri/xml_cdata.c +54 -0
data/ext/nokogiri/xml_cdata.h +9 -0
data/ext/nokogiri/xml_comment.c +52 -0
data/ext/nokogiri/xml_comment.h +9 -0
data/ext/nokogiri/xml_document.c +388 -0
data/ext/nokogiri/xml_document.h +24 -0
data/ext/nokogiri/xml_document_fragment.c +46 -0
data/ext/nokogiri/xml_document_fragment.h +10 -0
data/ext/nokogiri/xml_dtd.c +192 -0
data/ext/nokogiri/xml_dtd.h +10 -0
data/ext/nokogiri/xml_element_content.c +123 -0
data/ext/nokogiri/xml_element_content.h +10 -0
data/ext/nokogiri/xml_element_decl.c +69 -0
data/ext/nokogiri/xml_element_decl.h +9 -0
data/ext/nokogiri/xml_entity_decl.c +97 -0
data/ext/nokogiri/xml_entity_decl.h +10 -0
data/ext/nokogiri/xml_entity_reference.c +50 -0
data/ext/nokogiri/xml_entity_reference.h +9 -0
data/ext/nokogiri/xml_io.c +31 -0
data/ext/nokogiri/xml_io.h +11 -0
data/ext/nokogiri/xml_namespace.c +74 -0
data/ext/nokogiri/xml_namespace.h +12 -0
data/ext/nokogiri/xml_node.c +1060 -0
data/ext/nokogiri/xml_node.h +13 -0
data/ext/nokogiri/xml_node_set.c +397 -0
data/ext/nokogiri/xml_node_set.h +9 -0
data/ext/nokogiri/xml_processing_instruction.c +54 -0
data/ext/nokogiri/xml_processing_instruction.h +9 -0
data/ext/nokogiri/xml_reader.c +593 -0
data/ext/nokogiri/xml_reader.h +10 -0
data/ext/nokogiri/xml_relax_ng.c +159 -0
data/ext/nokogiri/xml_relax_ng.h +9 -0
data/ext/nokogiri/xml_sax_parser.c +286 -0
data/ext/nokogiri/xml_sax_parser.h +43 -0
data/ext/nokogiri/xml_sax_parser_context.c +155 -0
data/ext/nokogiri/xml_sax_parser_context.h +10 -0
data/ext/nokogiri/xml_sax_push_parser.c +114 -0
data/ext/nokogiri/xml_sax_push_parser.h +9 -0
data/ext/nokogiri/xml_schema.c +156 -0
data/ext/nokogiri/xml_schema.h +9 -0
data/ext/nokogiri/xml_syntax_error.c +261 -0
data/ext/nokogiri/xml_syntax_error.h +13 -0
data/ext/nokogiri/xml_text.c +48 -0
data/ext/nokogiri/xml_text.h +9 -0
data/ext/nokogiri/xml_xpath.c +53 -0
data/ext/nokogiri/xml_xpath.h +11 -0
data/ext/nokogiri/xml_xpath_context.c +239 -0
data/ext/nokogiri/xml_xpath_context.h +9 -0
data/ext/nokogiri/xslt_stylesheet.c +131 -0
data/ext/nokogiri/xslt_stylesheet.h +9 -0
data/lib/nokogiri.rb +116 -0
data/lib/nokogiri/css.rb +25 -0
data/lib/nokogiri/css/generated_parser.rb +646 -0
data/lib/nokogiri/css/generated_tokenizer.rb +142 -0
data/lib/nokogiri/css/node.rb +99 -0
data/lib/nokogiri/css/parser.rb +82 -0
data/lib/nokogiri/css/parser.y +227 -0
data/lib/nokogiri/css/syntax_error.rb +7 -0
data/lib/nokogiri/css/tokenizer.rb +7 -0
data/lib/nokogiri/css/tokenizer.rex +54 -0
data/lib/nokogiri/css/xpath_visitor.rb +162 -0
data/lib/nokogiri/decorators/slop.rb +33 -0
data/lib/nokogiri/ffi/html/document.rb +28 -0
data/lib/nokogiri/ffi/html/element_description.rb +85 -0
data/lib/nokogiri/ffi/html/entity_lookup.rb +16 -0
data/lib/nokogiri/ffi/html/sax/parser_context.rb +38 -0
data/lib/nokogiri/ffi/io_callbacks.rb +42 -0
data/lib/nokogiri/ffi/libxml.rb +356 -0
data/lib/nokogiri/ffi/structs/common_node.rb +26 -0
data/lib/nokogiri/ffi/structs/html_elem_desc.rb +24 -0
data/lib/nokogiri/ffi/structs/html_entity_desc.rb +13 -0
data/lib/nokogiri/ffi/structs/xml_alloc.rb +16 -0
data/lib/nokogiri/ffi/structs/xml_attr.rb +19 -0
data/lib/nokogiri/ffi/structs/xml_attribute.rb +27 -0
data/lib/nokogiri/ffi/structs/xml_buffer.rb +16 -0
data/lib/nokogiri/ffi/structs/xml_document.rb +108 -0
data/lib/nokogiri/ffi/structs/xml_dtd.rb +28 -0
data/lib/nokogiri/ffi/structs/xml_element.rb +26 -0
data/lib/nokogiri/ffi/structs/xml_element_content.rb +17 -0
data/lib/nokogiri/ffi/structs/xml_entity.rb +32 -0
data/lib/nokogiri/ffi/structs/xml_enumeration.rb +12 -0
data/lib/nokogiri/ffi/structs/xml_node.rb +28 -0
data/lib/nokogiri/ffi/structs/xml_node_set.rb +53 -0
data/lib/nokogiri/ffi/structs/xml_notation.rb +11 -0
data/lib/nokogiri/ffi/structs/xml_ns.rb +15 -0
data/lib/nokogiri/ffi/structs/xml_parser_context.rb +19 -0
data/lib/nokogiri/ffi/structs/xml_relax_ng.rb +14 -0
data/lib/nokogiri/ffi/structs/xml_sax_handler.rb +51 -0
data/lib/nokogiri/ffi/structs/xml_sax_push_parser_context.rb +15 -0
data/lib/nokogiri/ffi/structs/xml_schema.rb +13 -0
data/lib/nokogiri/ffi/structs/xml_syntax_error.rb +31 -0
data/lib/nokogiri/ffi/structs/xml_text_reader.rb +12 -0
data/lib/nokogiri/ffi/structs/xml_xpath_context.rb +37 -0
data/lib/nokogiri/ffi/structs/xml_xpath_object.rb +35 -0
data/lib/nokogiri/ffi/structs/xml_xpath_parser_context.rb +20 -0
data/lib/nokogiri/ffi/structs/xslt_stylesheet.rb +13 -0
data/lib/nokogiri/ffi/xml/attr.rb +41 -0
data/lib/nokogiri/ffi/xml/attribute_decl.rb +27 -0
data/lib/nokogiri/ffi/xml/cdata.rb +19 -0
data/lib/nokogiri/ffi/xml/comment.rb +18 -0
data/lib/nokogiri/ffi/xml/document.rb +135 -0
data/lib/nokogiri/ffi/xml/document_fragment.rb +21 -0
data/lib/nokogiri/ffi/xml/dtd.rb +69 -0
data/lib/nokogiri/ffi/xml/element_content.rb +43 -0
data/lib/nokogiri/ffi/xml/element_decl.rb +19 -0
data/lib/nokogiri/ffi/xml/entity_decl.rb +27 -0
data/lib/nokogiri/ffi/xml/entity_reference.rb +19 -0
data/lib/nokogiri/ffi/xml/namespace.rb +44 -0
data/lib/nokogiri/ffi/xml/node.rb +444 -0
data/lib/nokogiri/ffi/xml/node_set.rb +133 -0
data/lib/nokogiri/ffi/xml/processing_instruction.rb +20 -0
data/lib/nokogiri/ffi/xml/reader.rb +227 -0
data/lib/nokogiri/ffi/xml/relax_ng.rb +85 -0
data/lib/nokogiri/ffi/xml/sax/parser.rb +142 -0
data/lib/nokogiri/ffi/xml/sax/parser_context.rb +67 -0
data/lib/nokogiri/ffi/xml/sax/push_parser.rb +39 -0
data/lib/nokogiri/ffi/xml/schema.rb +92 -0
data/lib/nokogiri/ffi/xml/syntax_error.rb +91 -0
data/lib/nokogiri/ffi/xml/text.rb +18 -0
data/lib/nokogiri/ffi/xml/xpath.rb +19 -0
data/lib/nokogiri/ffi/xml/xpath_context.rb +135 -0
data/lib/nokogiri/ffi/xslt/stylesheet.rb +47 -0
data/lib/nokogiri/html.rb +35 -0
data/lib/nokogiri/html/builder.rb +35 -0
data/lib/nokogiri/html/document.rb +88 -0
data/lib/nokogiri/html/document_fragment.rb +15 -0
data/lib/nokogiri/html/element_description.rb +23 -0
data/lib/nokogiri/html/entity_lookup.rb +13 -0
data/lib/nokogiri/html/sax/parser.rb +48 -0
data/lib/nokogiri/html/sax/parser_context.rb +16 -0
data/lib/nokogiri/syntax_error.rb +4 -0
data/lib/nokogiri/version.rb +33 -0
data/lib/nokogiri/version_warning.rb +11 -0
data/lib/nokogiri/xml.rb +67 -0
data/lib/nokogiri/xml/attr.rb +14 -0
data/lib/nokogiri/xml/attribute_decl.rb +18 -0
data/lib/nokogiri/xml/builder.rb +405 -0
data/lib/nokogiri/xml/cdata.rb +11 -0
data/lib/nokogiri/xml/character_data.rb +7 -0
data/lib/nokogiri/xml/document.rb +131 -0
data/lib/nokogiri/xml/document_fragment.rb +69 -0
data/lib/nokogiri/xml/dtd.rb +11 -0
data/lib/nokogiri/xml/element_content.rb +36 -0
data/lib/nokogiri/xml/element_decl.rb +13 -0
data/lib/nokogiri/xml/entity_decl.rb +15 -0
data/lib/nokogiri/xml/fragment_handler.rb +71 -0
data/lib/nokogiri/xml/namespace.rb +13 -0
data/lib/nokogiri/xml/node.rb +665 -0
data/lib/nokogiri/xml/node/save_options.rb +42 -0
data/lib/nokogiri/xml/node_set.rb +307 -0
data/lib/nokogiri/xml/notation.rb +6 -0
data/lib/nokogiri/xml/parse_options.rb +85 -0
data/lib/nokogiri/xml/pp.rb +2 -0
data/lib/nokogiri/xml/pp/character_data.rb +18 -0
data/lib/nokogiri/xml/pp/node.rb +56 -0
data/lib/nokogiri/xml/processing_instruction.rb +8 -0
data/lib/nokogiri/xml/reader.rb +74 -0
data/lib/nokogiri/xml/relax_ng.rb +32 -0
data/lib/nokogiri/xml/sax.rb +4 -0
data/lib/nokogiri/xml/sax/document.rb +160 -0
data/lib/nokogiri/xml/sax/parser.rb +115 -0
data/lib/nokogiri/xml/sax/parser_context.rb +16 -0
data/lib/nokogiri/xml/sax/push_parser.rb +60 -0
data/lib/nokogiri/xml/schema.rb +61 -0
data/lib/nokogiri/xml/syntax_error.rb +38 -0
data/lib/nokogiri/xml/xpath.rb +10 -0
data/lib/nokogiri/xml/xpath/syntax_error.rb +8 -0
data/lib/nokogiri/xml/xpath_context.rb +16 -0
data/lib/nokogiri/xslt.rb +48 -0
data/lib/nokogiri/xslt/stylesheet.rb +25 -0
data/lib/xsd/xmlparser/nokogiri.rb +71 -0
data/tasks/test.rb +100 -0
data/test/css/test_nthiness.rb +159 -0
data/test/css/test_parser.rb +277 -0
data/test/css/test_tokenizer.rb +183 -0
data/test/css/test_xpath_visitor.rb +76 -0
data/test/ffi/test_document.rb +35 -0
data/test/files/2ch.html +108 -0
data/test/files/address_book.rlx +12 -0
data/test/files/address_book.xml +10 -0
data/test/files/bar/bar.xsd +4 -0
data/test/files/dont_hurt_em_why.xml +422 -0
data/test/files/exslt.xml +8 -0
data/test/files/exslt.xslt +35 -0
data/test/files/foo/foo.xsd +4 -0
data/test/files/po.xml +32 -0
data/test/files/po.xsd +66 -0
data/test/files/shift_jis.html +10 -0
data/test/files/shift_jis.xml +5 -0
data/test/files/snuggles.xml +3 -0
data/test/files/staff.dtd +10 -0
data/test/files/staff.xml +59 -0
data/test/files/staff.xslt +32 -0
data/test/files/tlm.html +850 -0
data/test/files/valid_bar.xml +2 -0
data/test/helper.rb +136 -0
data/test/html/sax/test_parser.rb +64 -0
data/test/html/sax/test_parser_context.rb +48 -0
data/test/html/test_builder.rb +164 -0
data/test/html/test_document.rb +390 -0
data/test/html/test_document_encoding.rb +77 -0
data/test/html/test_document_fragment.rb +132 -0
data/test/html/test_element_description.rb +94 -0
data/test/html/test_named_characters.rb +14 -0
data/test/html/test_node.rb +228 -0
data/test/html/test_node_encoding.rb +27 -0
data/test/test_convert_xpath.rb +135 -0
data/test/test_css_cache.rb +45 -0
data/test/test_gc.rb +15 -0
data/test/test_memory_leak.rb +77 -0
data/test/test_nokogiri.rb +134 -0
data/test/test_reader.rb +358 -0
data/test/test_xslt_transforms.rb +131 -0
data/test/xml/node/test_save_options.rb +20 -0
data/test/xml/node/test_subclass.rb +44 -0
data/test/xml/sax/test_parser.rb +307 -0
data/test/xml/sax/test_parser_context.rb +56 -0
data/test/xml/sax/test_push_parser.rb +131 -0
data/test/xml/test_attr.rb +38 -0
data/test/xml/test_attribute_decl.rb +82 -0
data/test/xml/test_builder.rb +167 -0
data/test/xml/test_cdata.rb +38 -0
data/test/xml/test_comment.rb +29 -0
data/test/xml/test_document.rb +607 -0
data/test/xml/test_document_encoding.rb +26 -0
data/test/xml/test_document_fragment.rb +138 -0
data/test/xml/test_dtd.rb +82 -0
data/test/xml/test_dtd_encoding.rb +33 -0
data/test/xml/test_element_content.rb +56 -0
data/test/xml/test_element_decl.rb +73 -0
data/test/xml/test_entity_decl.rb +83 -0
data/test/xml/test_entity_reference.rb +21 -0
data/test/xml/test_namespace.rb +68 -0
data/test/xml/test_node.rb +889 -0
data/test/xml/test_node_attributes.rb +34 -0
data/test/xml/test_node_encoding.rb +107 -0
data/test/xml/test_node_set.rb +531 -0
data/test/xml/test_parse_options.rb +52 -0
data/test/xml/test_processing_instruction.rb +30 -0
data/test/xml/test_reader_encoding.rb +126 -0
data/test/xml/test_relax_ng.rb +60 -0
data/test/xml/test_schema.rb +89 -0
data/test/xml/test_syntax_error.rb +27 -0
data/test/xml/test_text.rb +30 -0
data/test/xml/test_unparented_node.rb +381 -0
data/test/xml/test_xpath.rb +106 -0
metadata +430 -0

data/test/css/test_nthiness.rb ADDED

@@ -0,0 +1,159 @@
+require "helper"
+module Nokogiri
+  module CSS
+    class TestNthiness < Nokogiri::TestCase
+      def setup
+        super
+        doc = <<EOF
+<html>
+<table>
+  <tr><td>row1 </td></tr>
+  <tr><td>row2 </td></tr>
+  <tr><td>row3 </td></tr>
+  <tr><td>row4 </td></tr>
+  <tr><td>row5 </td></tr>
+  <tr><td>row6 </td></tr>
+  <tr><td>row7 </td></tr>
+  <tr><td>row8 </td></tr>
+  <tr><td>row9 </td></tr>
+  <tr><td>row10 </td></tr>
+  <tr><td>row11 </td></tr>
+  <tr><td>row12 </td></tr>
+  <tr><td>row13 </td></tr>
+  <tr><td>row14 </td></tr>
+</table>
+<div>
+  <b>bold1 </b>
+  <i>italic1 </i>
+  <b>bold2 </b>
+  <i>italic2 </i>
+  <p>para1 </p>
+  <b>bold3 </b>
+</div>
+<div>
+  <p>para2 </p>
+  <p>para3 </p>
+</div>
+<div>
+  <p>para4 </p>
+</div>
+<p class='empty'></p>
+<p class='not-empty'><b></b></p>
+</html>
+EOF
+        @parser = Nokogiri.HTML doc
+      end
+      def test_even
+        assert_result_rows [2,4,6,8,10,12,14], @parser.search("table/tr:nth(even)")
+      end
+      def test_odd
+        assert_result_rows [1,3,5,7,9,11,13], @parser.search("table/tr:nth(odd)")
+      end
+      def test_2n
+        assert_equal @parser.search("table/tr:nth(even)").inner_text, @parser.search("table/tr:nth(2n)").inner_text
+      end
+      def test_2np1
+        assert_equal @parser.search("table/tr:nth(odd)").inner_text, @parser.search("table/tr:nth(2n+1)").inner_text
+      end
+      def test_4np3
+        assert_result_rows [3,7,11], @parser.search("table/tr:nth(4n+3)")
+      end
+      def test_3np4
+        assert_result_rows [4,7,10,13], @parser.search("table/tr:nth(3n+4)")
+      end
+      def test_mnp3
+        assert_result_rows [1,2,3], @parser.search("table/tr:nth(-n+3)")
+      end
+      def test_np3
+        assert_result_rows [3,4,5,6,7,8,9,10,11,12,13,14], @parser.search("table/tr:nth(n+3)")
+      end
+      def test_first
+        assert_result_rows [1], @parser.search("table/tr:first")
+        assert_result_rows [1], @parser.search("table/tr:first()")
+      end
+      def test_last
+        assert_result_rows [14], @parser.search("table/tr:last")
+        assert_result_rows [14], @parser.search("table/tr:last()")
+      end
+      def test_first_child
+        assert_result_rows [1], @parser.search("div/b:first-child"), "bold"
+        assert_result_rows [1], @parser.search("table/tr:first-child")
+      end
+      def test_last_child
+        assert_result_rows [3], @parser.search("div/b:last-child"), "bold"
+        assert_result_rows [14], @parser.search("table/tr:last-child")
+      end
+      def test_first_of_type
+        assert_result_rows [1], @parser.search("table/tr:first-of-type")
+        assert_result_rows [1], @parser.search("div/b:first-of-type"), "bold"
+      end
+      def test_last_of_type
+        assert_result_rows [14], @parser.search("table/tr:last-of-type")
+        assert_result_rows [3], @parser.search("div/b:last-of-type"), "bold"
+      end
+      def test_only_of_type
+        assert_result_rows [1,4], @parser.search("div/p:only-of-type"), "para"
+      end
+      def test_only_child
+        assert_result_rows [4], @parser.search("div/p:only-child"), "para"
+      end
+      def test_empty
+        result = @parser.search("p:empty")
+        assert_equal 1, result.size, "unexpected number of rows returned: '#{result.inner_text}'"
+        assert_equal 'empty', result.first['class']
+      end
+      def test_parent
+        result = @parser.search("p:parent")
+        assert_equal 5, result.size
+        0.upto(3) do |j|
+          assert_equal "para#{j+1} ", result[j].inner_text
+        end
+        assert_equal "not-empty", result[4]['class']
+      end
+      def test_siblings
+        doc = <<-EOF
+<html><body><div>
+<p id="1">p1 </p>
+<p id="2">p2 </p>
+<p id="3">p3 </p>
+<p id="4">p4 </p>
+<p id="5">p5 </p>
+EOF
+        parser = Nokogiri.HTML doc
+        assert_equal 2, parser.search("#3 ~ p").size
+        assert_equal "p4 p5 ", parser.search("#3 ~ p").inner_text
+        assert_equal 0, parser.search("#5 ~ p").size
+        assert_equal 1, parser.search("#3 + p").size
+        assert_equal "p4 ", parser.search("#3 + p").inner_text
+        assert_equal 0, parser.search("#5 + p").size
+      end
+      def assert_result_rows intarray, result, word="row"
+        assert_equal intarray.size, result.size, "unexpected number of rows returned: '#{result.inner_text}'"
+        assert_equal intarray.map{|j| "#{word}#{j}"}.join(' '), result.inner_text.strip, result.inner_text
+      end
+    end
+  end
+end

data/test/css/test_parser.rb ADDED

@@ -0,0 +1,277 @@
+require "helper"
+module Nokogiri
+  module CSS
+    class TestParser < Nokogiri::TestCase
+      def setup
+        super
+        @parser = Nokogiri::CSS::Parser.new
+      end
+      def test_extra_single_quote
+        assert_raises(CSS::SyntaxError) { @parser.parse("'") }
+      end
+      def test_syntax_error_raised
+        assert_raises(CSS::SyntaxError) { @parser.parse("a[x=]") }
+      end
+      def test_find_by_type
+        ast = @parser.parse("a:nth-child(2)").first
+        matches = ast.find_by_type(
+          [:CONDITIONAL_SELECTOR,
+            [:ELEMENT_NAME],
+            [:PSEUDO_CLASS,
+              [:FUNCTION]
+            ]
+          ]
+        )
+        assert_equal(1, matches.length)
+        assert_equal(ast, matches.first)
+      end
+      def test_to_type
+        ast = @parser.parse("a:nth-child(2)").first
+        assert_equal(
+          [:CONDITIONAL_SELECTOR,
+            [:ELEMENT_NAME],
+            [:PSEUDO_CLASS,
+              [:FUNCTION]
+            ]
+          ], ast.to_type
+        )
+      end
+      def test_to_a
+        asts = @parser.parse("a:nth-child(2)")
+        assert_equal(
+          [:CONDITIONAL_SELECTOR,
+            [:ELEMENT_NAME, ["a"]],
+            [:PSEUDO_CLASS,
+              [:FUNCTION, ["nth-child("], ["2"]]
+            ]
+          ], asts.first.to_a
+        )
+      end
+      def test_dashmatch
+        assert_xpath  "//a[@class = 'bar' or starts-with(@class, concat('bar', '-'))]",
+                      @parser.parse("a[@class|='bar']")
+        assert_xpath  "//a[@class = 'bar' or starts-with(@class, concat('bar', '-'))]",
+                      @parser.parse("a[@class |= 'bar']")
+      end
+      def test_includes
+        assert_xpath  "//a[contains(concat(\" \", @class, \" \"),concat(\" \", 'bar', \" \"))]",
+                      @parser.parse("a[@class~='bar']")
+        assert_xpath  "//a[contains(concat(\" \", @class, \" \"),concat(\" \", 'bar', \" \"))]",
+                      @parser.parse("a[@class ~= 'bar']")
+      end
+      def test_function_with_arguments
+        assert_xpath  "//*[position() = 2 and self::a]",
+                      @parser.parse("a[2]")
+        assert_xpath  "//*[position() = 2 and self::a]",
+                      @parser.parse("a:nth-child(2)")
+      end
+      def test_carrot
+        assert_xpath  "//a[starts-with(@id, 'Boing')]",
+                      @parser.parse("a[id^='Boing']")
+        assert_xpath  "//a[starts-with(@id, 'Boing')]",
+                      @parser.parse("a[id ^= 'Boing']")
+      end
+      def test_suffix_match
+        assert_xpath "//a[substring(@id, string-length(@id) - string-length('Boing') + 1, string-length('Boing')) = 'Boing']",
+                      @parser.parse("a[id$='Boing']")
+        assert_xpath "//a[substring(@id, string-length(@id) - string-length('Boing') + 1, string-length('Boing')) = 'Boing']",
+                      @parser.parse("a[id $= 'Boing']")
+      end
+      def test_attributes_with_at
+        ## This is non standard CSS
+        assert_xpath  "//a[@id = 'Boing']",
+                      @parser.parse("a[@id='Boing']")
+        assert_xpath  "//a[@id = 'Boing']",
+                      @parser.parse("a[@id = 'Boing']")
+      end
+      def test_attributes_with_at_and_stuff
+        ## This is non standard CSS
+        assert_xpath  "//a[@id = 'Boing']//div",
+                      @parser.parse("a[@id='Boing'] div")
+      end
+      def test_not_equal
+        ## This is non standard CSS
+        assert_xpath  "//a[child::text() != 'Boing']",
+                      @parser.parse("a[text()!='Boing']")
+        assert_xpath  "//a[child::text() != 'Boing']",
+                      @parser.parse("a[text() != 'Boing']")
+      end
+      def test_function
+        ## This is non standard CSS
+        assert_xpath  "//a[child::text()]",
+                      @parser.parse("a[text()]")
+        ## This is non standard CSS
+        assert_xpath  "//child::text()",
+                      @parser.parse("text()")
+        ## This is non standard CSS
+        assert_xpath  "//a[contains(child::text(), 'Boing')]",
+                      @parser.parse("a[text()*='Boing']")
+        assert_xpath  "//a[contains(child::text(), 'Boing')]",
+                      @parser.parse("a[text() *= 'Boing']")
+        ## This is non standard CSS
+        assert_xpath  "//script//comment()",
+                      @parser.parse("script comment()")
+      end
+      def test_nonstandard_nth_selectors
+        ## These are non standard CSS
+        assert_xpath '//a[position() = 99]', @parser.parse('a:eq(99)')
+        assert_xpath '//a[position() = 1]', @parser.parse('a:first') # no parens
+        assert_xpath '//a[position() = last()]', @parser.parse('a:last') # no parens
+        assert_xpath '//a[position() = 99]', @parser.parse('a:nth(99)')
+        assert_xpath '//a[position() = 1]', @parser.parse('a:first()')
+        assert_xpath '//a[position() = last()]', @parser.parse('a:last()')
+        assert_xpath '//a[node()]', @parser.parse('a:parent')
+      end
+      def test_standard_nth_selectors
+        assert_xpath '//a[position() = 99]', @parser.parse('a:nth-of-type(99)')
+        assert_xpath '//a[position() = 1]', @parser.parse('a:first-of-type()')
+        assert_xpath '//a[position() = last()]', @parser.parse('a:last-of-type()')
+        assert_xpath '//a[position() = 1]', @parser.parse('a:first-of-type') # no parens
+        assert_xpath '//a[position() = last()]', @parser.parse('a:last-of-type') # no parens
+        assert_xpath '//a[position() = last() - 99]', @parser.parse('a:nth-last-of-type(99)')
+        assert_xpath '//a[position() = last() - 99]', @parser.parse('a:nth-last-of-type(99)')
+      end
+      def test_nth_child_selectors
+        assert_xpath '//*[position() = 1 and self::a]', @parser.parse('a:first-child')
+        assert_xpath '//*[position() = last() and self::a]', @parser.parse('a:last-child')
+        assert_xpath '//*[position() = 99 and self::a]', @parser.parse('a:nth-child(99)')
+        assert_xpath '//*[position() = last() - 99 and self::a]', @parser.parse('a:nth-last-child(99)')
+      end
+      def test_miscellaneous_selectors
+        assert_xpath '//*[last() = 1 and self::a]',
+          @parser.parse('a:only-child')
+        assert_xpath '//a[last() = 1]', @parser.parse('a:only-of-type')
+        assert_xpath '//a[not(node())]', @parser.parse('a:empty')
+      end
+      def test_nth_a_n_plus_b
+        assert_xpath '//a[(position() mod 2) = 0]', @parser.parse('a:nth-of-type(2n)')
+        assert_xpath '//a[(position() >= 1) and (((position()-1) mod 2) = 0)]', @parser.parse('a:nth-of-type(2n+1)')
+        assert_xpath '//a[(position() mod 2) = 0]', @parser.parse('a:nth-of-type(even)')
+        assert_xpath '//a[(position() >= 1) and (((position()-1) mod 2) = 0)]', @parser.parse('a:nth-of-type(odd)')
+        assert_xpath '//a[(position() >= 3) and (((position()-3) mod 4) = 0)]', @parser.parse('a:nth-of-type(4n+3)')
+        assert_xpath '//a[(position() <= 3) and (((position()-3) mod 1) = 0)]', @parser.parse('a:nth-of-type(-1n+3)')
+        assert_xpath '//a[(position() <= 3) and (((position()-3) mod 1) = 0)]', @parser.parse('a:nth-of-type(-n+3)')
+        assert_xpath '//a[(position() >= 3) and (((position()-3) mod 1) = 0)]', @parser.parse('a:nth-of-type(1n+3)')
+        assert_xpath '//a[(position() >= 3) and (((position()-3) mod 1) = 0)]', @parser.parse('a:nth-of-type(n+3)')
+      end
+      def test_preceding_selector
+        assert_xpath  "//F[preceding-sibling::E]",
+                      @parser.parse("E ~ F")
+      end
+      def test_direct_preceding_selector
+        assert_xpath  "//E/following-sibling::*[1]/self::F",
+                      @parser.parse("E + F")
+      end
+      def test_attribute
+        assert_xpath  "//h1[@a = 'Tender Lovemaking']",
+                      @parser.parse("h1[a='Tender Lovemaking']")
+      end
+      def test_id
+        assert_xpath "//*[@id = 'foo']", @parser.parse('#foo')
+      end
+      def test_pseudo_class_no_ident
+        assert_xpath "//*[link(.)]", @parser.parse(':link')
+      end
+      def test_pseudo_class
+        assert_xpath "//a[link(.)]", @parser.parse('a:link')
+        assert_xpath "//a[visited(.)]", @parser.parse('a:visited')
+        assert_xpath "//a[hover(.)]", @parser.parse('a:hover')
+        assert_xpath "//a[active(.)]", @parser.parse('a:active')
+        assert_xpath  "//a[active(.) and contains(concat(' ', @class, ' '), ' foo ')]",
+                      @parser.parse('a:active.foo')
+      end
+      def test_star
+        assert_xpath "//*", @parser.parse('*')
+        assert_xpath "//*[contains(concat(' ', @class, ' '), ' pastoral ')]",
+                      @parser.parse('*.pastoral')
+      end
+      def test_class
+        assert_xpath  "//*[contains(concat(' ', @class, ' '), ' a ') and contains(concat(' ', @class, ' '), ' b ')]",
+                      @parser.parse('.a.b')
+        assert_xpath  "//*[contains(concat(' ', @class, ' '), ' awesome ')]",
+                      @parser.parse('.awesome')
+        assert_xpath  "//foo[contains(concat(' ', @class, ' '), ' awesome ')]",
+                      @parser.parse('foo.awesome')
+        assert_xpath  "//foo//*[contains(concat(' ', @class, ' '), ' awesome ')]",
+                      @parser.parse('foo .awesome')
+      end
+      def test_not_so_simple_not
+        assert_xpath "//*[@id = 'p' and not(contains(concat(' ', @class, ' '), ' a '))]",
+                     @parser.parse('#p:not(.a)')
+        assert_xpath "//p[contains(concat(' ', @class, ' '), ' a ') and not(contains(concat(' ', @class, ' '), ' b '))]",
+                     @parser.parse('p.a:not(.b)')
+        assert_xpath "//p[@a = 'foo' and not(contains(concat(' ', @class, ' '), ' b '))]",
+                     @parser.parse("p[a='foo']:not(.b)")
+      end
+      def test_ident
+        assert_xpath '//x', @parser.parse('x')
+      end
+      def test_parse_space
+        assert_xpath '//x//y', @parser.parse('x y')
+      end
+      def test_parse_descendant
+        assert_xpath '//x/y', @parser.parse('x > y')
+      end
+      def test_parse_slash
+        ## This is non standard CSS
+        assert_xpath '//x/y', @parser.parse('x/y')
+      end
+      def test_parse_doubleslash
+        ## This is non standard CSS
+        assert_xpath '//x//y', @parser.parse('x//y')
+      end
+      def test_multi_path
+        assert_xpath ['//x/y', '//y/z'], @parser.parse('x > y, y > z')
+        assert_xpath ['//x/y', '//y/z'], @parser.parse('x > y,y > z')
+        ###
+        # TODO: should we make this work?
+        # assert_xpath ['//x/y', '//y/z'], @parser.parse('x > y | y > z')
+      end
+      def assert_xpath expecteds, asts
+        expecteds = [expecteds].flatten
+        expecteds.zip(asts).each do |expected, actual|
+          assert_equal expected, actual.to_xpath
+        end
+      end
+    end
+  end
+end

data/test/css/test_tokenizer.rb ADDED

@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*-
+require "helper"
+module Nokogiri
+  module CSS
+    class TestTokenizer < Nokogiri::TestCase
+      def setup
+        super
+        @scanner = Nokogiri::CSS::Tokenizer.new
+      end
+      def test_unicode
+        @scanner.scan("a日本語")
+        assert_tokens([[:IDENT, 'a日本語']], @scanner)
+      end
+      def test_tokenize_bad_single_quote
+        @scanner.scan("'")
+        assert_tokens([["'", "'"]], @scanner)
+      end
+      def test_not_equal
+        @scanner.scan("h1[a!='Tender Lovemaking']")
+        assert_tokens([ [:IDENT, 'h1'],
+                        [:LSQUARE, '['],
+                        [:IDENT, 'a'],
+                        [:NOT_EQUAL, '!='],
+                        [:STRING, "'Tender Lovemaking'"],
+                        [:RSQUARE, ']'],
+        ], @scanner)
+      end
+      def test_negation
+        @scanner.scan("p:not(.a)")
+        assert_tokens([ [:IDENT, 'p'],
+                        [:NOT, ':not('],
+                        ['.', '.'],
+                        [:IDENT, 'a'],
+                        [:RPAREN, ')'],
+        ], @scanner)
+      end
+      def test_function
+        @scanner.scan("script comment()")
+        assert_tokens([ [:IDENT, 'script'],
+                        [:S, ' '],
+                        [:FUNCTION, 'comment('],
+                        [:RPAREN, ')'],
+        ], @scanner)
+      end
+      def test_preceding_selector
+        @scanner.scan("E ~ F")
+        assert_tokens([ [:IDENT, 'E'],
+                        [:TILDE, ' ~ '],
+                        [:IDENT, 'F'],
+        ], @scanner)
+      end
+      def test_scan_attribute_string
+        @scanner.scan("h1[a='Tender Lovemaking']")
+        assert_tokens([ [:IDENT, 'h1'],
+                        [:LSQUARE, '['],
+                        [:IDENT, 'a'],
+                        [:EQUAL, '='],
+                        [:STRING, "'Tender Lovemaking'"],
+                        [:RSQUARE, ']'],
+        ], @scanner)
+        @scanner.scan('h1[a="Tender Lovemaking"]')
+        assert_tokens([ [:IDENT, 'h1'],
+                        [:LSQUARE, '['],
+                        [:IDENT, 'a'],
+                        [:EQUAL, '='],
+                        [:STRING, '"Tender Lovemaking"'],
+                        [:RSQUARE, ']'],
+        ], @scanner)
+      end
+      def test_scan_id
+        @scanner.scan('#foo')
+        assert_tokens([ [:HASH, '#foo'] ], @scanner)
+      end
+      def test_scan_pseudo
+        @scanner.scan('a:visited')
+        assert_tokens([ [:IDENT, 'a'],
+                        [':', ':'],
+                        [:IDENT, 'visited']
+        ], @scanner)
+      end
+      def test_scan_star
+        @scanner.scan('*')
+        assert_tokens([ ['*', '*'], ], @scanner)
+      end
+      def test_scan_class
+        @scanner.scan('x.awesome')
+        assert_tokens([ [:IDENT, 'x'],
+                        ['.', '.'],
+                        [:IDENT, 'awesome'],
+        ], @scanner)
+      end
+      def test_scan_greater
+        @scanner.scan('x > y')
+        assert_tokens([ [:IDENT, 'x'],
+                        [:GREATER, ' > '],
+                        [:IDENT, 'y']
+        ], @scanner)
+      end
+      def test_scan_slash
+        @scanner.scan('x/y')
+        assert_tokens([ [:IDENT, 'x'],
+                        [:SLASH, '/'],
+                        [:IDENT, 'y']
+        ], @scanner)
+      end
+      def test_scan_doubleslash
+        @scanner.scan('x//y')
+        assert_tokens([ [:IDENT, 'x'],
+                        [:DOUBLESLASH, '//'],
+                        [:IDENT, 'y']
+        ], @scanner)
+      end
+      def test_scan_function_selector
+        @scanner.scan('x:eq(0)')
+        assert_tokens([ [:IDENT, 'x'],
+                        [':', ':'],
+                        [:FUNCTION, 'eq('],
+                        [:NUMBER, "0"],
+                        [:RPAREN, ')'],
+        ], @scanner)
+      end
+      def test_scan_an_plus_b
+        @scanner.scan('x:nth-child(5n+3)')
+        assert_tokens([ [:IDENT, 'x'],
+                        [':', ':'],
+                        [:FUNCTION, 'nth-child('],
+                        [:NUMBER, '5'],
+                        [:IDENT, 'n'],
+                        [:PLUS, '+'],
+                        [:NUMBER, '3'],
+                        [:RPAREN, ')'],
+        ], @scanner)
+        @scanner.scan('x:nth-child(-1n+3)')
+        assert_tokens([ [:IDENT, 'x'],
+                        [':', ':'],
+                        [:FUNCTION, 'nth-child('],
+                        [:NUMBER, '-1'],
+                        [:IDENT, 'n'],
+                        [:PLUS, '+'],
+                        [:NUMBER, '3'],
+                        [:RPAREN, ')'],
+        ], @scanner)
+        @scanner.scan('x:nth-child(-n+3)')
+        assert_tokens([ [:IDENT, 'x'],
+                        [':', ':'],
+                        [:FUNCTION, 'nth-child('],
+                        [:IDENT, '-n'],
+                        [:PLUS, '+'],
+                        [:NUMBER, '3'],
+                        [:RPAREN, ')'],
+        ], @scanner)
+      end
+      def assert_tokens(tokens, scanner)
+        toks = []
+        while tok = @scanner.next_token
+          toks << tok
+        end
+        assert_equal(tokens, toks)
+      end
+    end
+  end
+end