nokogiri 1.3.2-x86-mswin32 → 1.3.3-x86-mswin32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of nokogiri might be problematic. Click here for more details.

Files changed (68) hide show
  1. data/CHANGELOG.ja.rdoc +25 -4
  2. data/CHANGELOG.rdoc +20 -0
  3. data/Manifest.txt +2 -0
  4. data/Rakefile +67 -24
  5. data/ext/nokogiri/extconf.rb +16 -9
  6. data/ext/nokogiri/html_document.c +0 -2
  7. data/ext/nokogiri/nokogiri.c +2 -0
  8. data/ext/nokogiri/nokogiri.h +3 -4
  9. data/ext/nokogiri/xml_document.c +30 -23
  10. data/ext/nokogiri/xml_document.h +3 -2
  11. data/ext/nokogiri/xml_dtd.c +4 -0
  12. data/ext/nokogiri/xml_dtd.h +2 -0
  13. data/ext/nokogiri/xml_node.c +28 -9
  14. data/ext/nokogiri/xml_reader.c +0 -7
  15. data/ext/nokogiri/xml_relax_ng.c +7 -1
  16. data/ext/nokogiri/xml_sax_parser.c +2 -0
  17. data/lib/action-nokogiri.rb +2 -0
  18. data/lib/nokogiri.rb +9 -3
  19. data/lib/nokogiri/1.8/nokogiri.so +0 -0
  20. data/lib/nokogiri/1.9/nokogiri.so +0 -0
  21. data/lib/nokogiri/css/generated_tokenizer.rb +80 -82
  22. data/lib/nokogiri/css/tokenizer.rb +1 -5
  23. data/lib/nokogiri/decorators/hpricot/node_set.rb +1 -1
  24. data/lib/nokogiri/ffi/structs/common_node.rb +1 -1
  25. data/lib/nokogiri/ffi/structs/xml_document.rb +1 -1
  26. data/lib/nokogiri/ffi/xml/document.rb +15 -4
  27. data/lib/nokogiri/ffi/xml/node.rb +85 -63
  28. data/lib/nokogiri/ffi/xml/reader.rb +4 -15
  29. data/lib/nokogiri/ffi/xml/relax_ng.rb +3 -1
  30. data/lib/nokogiri/hpricot.rb +30 -0
  31. data/lib/nokogiri/html/document.rb +3 -1
  32. data/lib/nokogiri/html/document_fragment.rb +1 -1
  33. data/lib/nokogiri/html/sax/parser.rb +2 -1
  34. data/lib/nokogiri/version.rb +1 -1
  35. data/lib/nokogiri/xml/builder.rb +44 -1
  36. data/lib/nokogiri/xml/document.rb +8 -1
  37. data/lib/nokogiri/xml/document_fragment.rb +1 -1
  38. data/lib/nokogiri/xml/fragment_handler.rb +4 -7
  39. data/lib/nokogiri/xml/node.rb +9 -6
  40. data/lib/nokogiri/xml/node_set.rb +7 -0
  41. data/lib/nokogiri/xml/parse_options.rb +1 -1
  42. data/test/css/test_nthiness.rb +2 -3
  43. data/test/ffi/test_document.rb +6 -6
  44. data/test/files/2ch.html +108 -0
  45. data/test/files/shift_jis.xml +5 -0
  46. data/test/helper.rb +3 -0
  47. data/test/hpricot/test_alter.rb +9 -9
  48. data/test/hpricot/test_builder.rb +2 -2
  49. data/test/hpricot/test_parser.rb +70 -146
  50. data/test/hpricot/test_paths.rb +2 -2
  51. data/test/hpricot/test_preserved.rb +2 -2
  52. data/test/hpricot/test_xml.rb +3 -3
  53. data/test/html/sax/test_parser.rb +12 -0
  54. data/test/html/test_builder.rb +6 -4
  55. data/test/html/test_document.rb +7 -0
  56. data/test/html/test_document_encoding.rb +17 -0
  57. data/test/html/test_document_fragment.rb +12 -0
  58. data/test/html/test_node.rb +5 -2
  59. data/test/test_convert_xpath.rb +1 -50
  60. data/test/test_css_cache.rb +1 -12
  61. data/test/test_nokogiri.rb +7 -0
  62. data/test/test_reader.rb +14 -0
  63. data/test/xml/test_document.rb +44 -0
  64. data/test/xml/test_document_fragment.rb +12 -0
  65. data/test/xml/test_node.rb +10 -2
  66. data/test/xml/test_node_encoding.rb +23 -0
  67. data/test/xml/test_node_set.rb +10 -0
  68. metadata +48 -46
@@ -113,6 +113,8 @@ static VALUE validate(VALUE self, VALUE document)
113
113
  return error_list;
114
114
  }
115
115
 
116
+ VALUE cNokogiriXmlDtd;
117
+
116
118
  void init_xml_dtd()
117
119
  {
118
120
  VALUE nokogiri = rb_define_module("Nokogiri");
@@ -124,6 +126,8 @@ void init_xml_dtd()
124
126
  */
125
127
  VALUE klass = rb_define_class_under(xml, "DTD", node);
126
128
 
129
+ cNokogiriXmlDtd = klass;
130
+
127
131
  rb_define_method(klass, "notations", notations, 0);
128
132
  rb_define_method(klass, "elements", elements, 0);
129
133
  rb_define_method(klass, "entities", entities, 0);
@@ -3,6 +3,8 @@
3
3
 
4
4
  #include <nokogiri.h>
5
5
 
6
+ extern VALUE cNokogiriXmlDtd;
7
+
6
8
  void init_xml_dtd();
7
9
 
8
10
  #endif
@@ -12,7 +12,10 @@ static void debug_node_dealloc(xmlNodePtr x)
12
12
 
13
13
  static void mark(xmlNodePtr node)
14
14
  {
15
- rb_gc_mark(DOC_RUBY_OBJECT(node->doc));
15
+ // it's OK if the document isn't fully realized (as in XML::Reader).
16
+ // see http://github.com/tenderlove/nokogiri/issues/closed/#issue/95
17
+ if (DOC_RUBY_OBJECT_TEST(node->doc) && DOC_RUBY_OBJECT(node->doc))
18
+ rb_gc_mark(DOC_RUBY_OBJECT(node->doc));
16
19
  }
17
20
 
18
21
  /* :nodoc: */
@@ -60,6 +63,17 @@ static VALUE reparent_node_with(VALUE node_obj, VALUE other_obj, node_other_func
60
63
  Data_Get_Struct(node_obj, xmlNode, node);
61
64
  Data_Get_Struct(other_obj, xmlNode, other);
62
65
 
66
+ // If a document fragment is added, we need to reparent all of it's children
67
+ if(node->type == XML_DOCUMENT_FRAG_NODE)
68
+ {
69
+ xmlNodePtr child = node->children;
70
+ while(NULL != child) {
71
+ reparent_node_with(Nokogiri_wrap_xml_node((VALUE)NULL, child), other_obj, func);
72
+ child = child->next;
73
+ }
74
+ return node_obj;
75
+ }
76
+
63
77
  if (node->doc == other->doc) {
64
78
  xmlUnlinkNode(node) ;
65
79
  if ( node->type == XML_TEXT_NODE
@@ -673,6 +687,8 @@ static VALUE native_write_to(
673
687
 
674
688
  xmlIndentTreeOutput = 1;
675
689
 
690
+ const char * before_indent = xmlTreeIndentString;
691
+
676
692
  xmlTreeIndentString = StringValuePtr(indent_string);
677
693
 
678
694
  xmlSaveCtxtPtr savectx = xmlSaveToIO(
@@ -685,6 +701,8 @@ static VALUE native_write_to(
685
701
 
686
702
  xmlSaveTree(savectx, node);
687
703
  xmlSaveClose(savectx);
704
+
705
+ xmlTreeIndentString = before_indent;
688
706
  return io;
689
707
  }
690
708
 
@@ -754,7 +772,10 @@ static VALUE new(int argc, VALUE *argv, VALUE klass)
754
772
  node->doc = doc->doc;
755
773
  NOKOGIRI_ROOT_NODE(node);
756
774
 
757
- VALUE rb_node = Nokogiri_wrap_xml_node(klass, node);
775
+ VALUE rb_node = Nokogiri_wrap_xml_node(
776
+ klass == cNokogiriXmlNode ? (VALUE)NULL : klass,
777
+ node
778
+ );
758
779
  rb_funcall2(rb_node, rb_intern("initialize"), argc, argv);
759
780
 
760
781
  if(rb_block_given_p()) rb_yield(rb_node);
@@ -774,9 +795,6 @@ static VALUE dump_html(VALUE self)
774
795
  xmlNodePtr node ;
775
796
  Data_Get_Struct(self, xmlNode, node);
776
797
 
777
- if(node->doc->type == XML_DOCUMENT_NODE)
778
- return rb_funcall(self, rb_intern("to_xml"), 0);
779
-
780
798
  buf = xmlBufferCreate() ;
781
799
  htmlNodeDump(buf, node->doc, node);
782
800
  VALUE html = NOKOGIRI_STR_NEW2(buf->content, node->doc->encoding);
@@ -845,7 +863,7 @@ VALUE Nokogiri_wrap_xml_node(VALUE klass, xmlNodePtr node)
845
863
  klass = cNokogiriXmlCData;
846
864
  break;
847
865
  case XML_DTD_NODE:
848
- klass = rb_const_get(mNokogiriXml, rb_intern("DTD"));
866
+ klass = cNokogiriXmlDtd;
849
867
  break;
850
868
  default:
851
869
  klass = cNokogiriXmlNode;
@@ -856,13 +874,14 @@ VALUE Nokogiri_wrap_xml_node(VALUE klass, xmlNodePtr node)
856
874
  node->_private = (void *)rb_node;
857
875
 
858
876
  if (DOC_RUBY_OBJECT_TEST(node->doc) && DOC_RUBY_OBJECT(node->doc)) {
877
+ // it's OK if the document isn't fully realized (as in XML::Reader).
878
+ // see http://github.com/tenderlove/nokogiri/issues/closed/#issue/95
859
879
  document = DOC_RUBY_OBJECT(node->doc);
860
880
  node_cache = DOC_NODE_CACHE(node->doc);
881
+ rb_ary_push(node_cache, rb_node);
882
+ rb_funcall(document, rb_intern("decorate"), 1, rb_node);
861
883
  }
862
884
 
863
- rb_ary_push(node_cache, rb_node);
864
- rb_funcall(document, rb_intern("decorate"), 1, rb_node);
865
-
866
885
  return rb_node ;
867
886
  }
868
887
 
@@ -163,13 +163,6 @@ static VALUE attribute_nodes(VALUE self)
163
163
  xmlNodePtr ptr = xmlTextReaderExpand(reader);
164
164
  if(ptr == NULL) return Qnil;
165
165
 
166
- // FIXME I'm not sure if this is correct..... I don't really like pointing
167
- // at this document, but I have to because of the assertions in
168
- // the node wrapping code.
169
- if(! DOC_RUBY_OBJECT_TEST(ptr->doc)) {
170
- VALUE rb_doc = Nokogiri_wrap_xml_document(cNokogiriXmlDocument, ptr->doc);
171
- RDATA(rb_doc)->dfree = NULL;
172
- }
173
166
  VALUE enc = rb_iv_get(self, "@encoding");
174
167
 
175
168
  if(enc != Qnil && NULL == ptr->doc->encoding) {
@@ -120,7 +120,9 @@ static VALUE from_document(VALUE klass, VALUE document)
120
120
  xmlRelaxNGPtr schema = xmlRelaxNGParse(ctx);
121
121
 
122
122
  xmlSetStructuredErrorFunc(NULL, NULL);
123
- xmlRelaxNGFreeParserCtxt(ctx);
123
+ if (! is_2_6_16()) {
124
+ xmlRelaxNGFreeParserCtxt(ctx);
125
+ }
124
126
 
125
127
  if(NULL == schema) {
126
128
  xmlErrorPtr error = xmlGetLastError();
@@ -132,6 +134,10 @@ static VALUE from_document(VALUE klass, VALUE document)
132
134
  return Qnil;
133
135
  }
134
136
 
137
+ if (is_2_6_16()) {
138
+ xmlRelaxNGFreeParserCtxt(ctx);
139
+ }
140
+
135
141
  VALUE rb_schema = Data_Wrap_Struct(klass, 0, dealloc, schema);
136
142
  rb_iv_set(rb_schema, "@errors", errors);
137
143
 
@@ -310,6 +310,8 @@ static VALUE allocate(VALUE klass)
310
310
  {
311
311
  xmlSAXHandlerPtr handler = calloc(1, sizeof(xmlSAXHandler));
312
312
 
313
+ xmlSetStructuredErrorFunc(NULL, NULL);
314
+
313
315
  handler->startDocument = start_document;
314
316
  handler->endDocument = end_document;
315
317
  handler->startElement = start_element;
@@ -18,6 +18,7 @@ module ActionController
18
18
  # Get your response as a Nokogiri::XML::Document using the
19
19
  # Nokogiri.HTML parser
20
20
  def html(flavor=nil)
21
+ warn "@response.html is deprecated and will be removed in nokogiri 1.4.0"
21
22
  if flavor == :hpricot
22
23
  @_nokogiri_html_hpricot ||= Nokogiri::Hpricot(body)
23
24
  else
@@ -29,6 +30,7 @@ module ActionController
29
30
  # Get your response as a Nokogiri::XML::Document using the
30
31
  # Nokogiri.XML parser
31
32
  def xml
33
+ warn "@response.html is deprecated and will be removed in nokogiri 1.4.0"
32
34
  @_nokogiri_xml ||= Nokogiri::XML(body)
33
35
  end
34
36
 
@@ -54,10 +54,16 @@ module Nokogiri
54
54
  # Parse an HTML or XML document. +string+ contains the document.
55
55
  def parse string, url = nil, encoding = nil, options = nil
56
56
  doc =
57
- if string =~ /^\s*<[^Hh>]*html/i # Probably html
58
- Nokogiri::HTML::Document.parse(string, url, encoding, options || XML::ParseOptions::DEFAULT_HTML)
57
+ if string.respond_to?(:read) ||
58
+ string =~ /^\s*<[^Hh>]*html/i # Probably html
59
+ Nokogiri::HTML(
60
+ string,
61
+ url,
62
+ encoding, options || XML::ParseOptions::DEFAULT_HTML
63
+ )
59
64
  else
60
- Nokogiri::XML::Document.parse(string, url, encoding, options || XML::ParseOptions::DEFAULT_XML)
65
+ Nokogiri::XML(string, url, encoding,
66
+ options || XML::ParseOptions::DEFAULT_XML)
61
67
  end
62
68
  yield doc if block_given?
63
69
  doc
Binary file
Binary file
@@ -1,6 +1,6 @@
1
1
  #--
2
2
  # DO NOT MODIFY!!!!
3
- # This file is automatically generated by rex 1.0.3
3
+ # This file is automatically generated by rex 1.0.4
4
4
  # from lexical definition file "lib/nokogiri/css/tokenizer.rex".
5
5
  #++
6
6
 
@@ -11,133 +11,131 @@ class GeneratedTokenizer < GeneratedParser
11
11
 
12
12
  class ScanError < StandardError ; end
13
13
 
14
- attr_reader :lineno
15
- attr_reader :filename
14
+ attr_reader :lineno
15
+ attr_reader :filename
16
+ attr_accessor :state
16
17
 
17
- def scan_setup ; end
18
+ def scan_setup(str)
19
+ @ss = StringScanner.new(str)
20
+ @lineno = 1
21
+ @state = nil
22
+ end
18
23
 
19
- def action &block
24
+ def action(&block)
20
25
  yield
21
26
  end
22
27
 
23
- def scan_str( str )
24
- scan_evaluate str
28
+ def scan_str(str)
29
+ scan_setup(str)
25
30
  do_parse
26
31
  end
27
32
 
28
33
  def load_file( filename )
29
34
  @filename = filename
30
35
  open(filename, "r") do |f|
31
- scan_evaluate f.read
36
+ scan_setup(f.read)
32
37
  end
33
38
  end
34
39
 
35
40
  def scan_file( filename )
36
- load_file filename
41
+ load_file(filename)
37
42
  do_parse
38
43
  end
39
44
 
40
- def next_token
41
- @rex_tokens.shift
42
- end
43
45
 
44
- def scan_evaluate( str )
45
- scan_setup
46
- @rex_tokens = []
47
- @lineno = 1
48
- ss = StringScanner.new(str)
49
- state = nil
50
- until ss.eos?
51
- text = ss.peek(1)
52
- @lineno += 1 if text == "\n"
53
- case state
54
- when nil
55
- case
56
- when (text = ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*\(\s*/))
57
- @rex_tokens.push action { [:FUNCTION, text] }
46
+ def next_token
47
+ return if @ss.eos?
58
48
 
59
- when (text = ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*/))
60
- @rex_tokens.push action { [:IDENT, text] }
49
+ text = @ss.peek(1)
50
+ @lineno += 1 if text == "\n"
51
+ token = case @state
52
+ when nil
53
+ case
54
+ when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*\(\s*/))
55
+ action { [:FUNCTION, text] }
61
56
 
62
- when (text = ss.scan(/\#([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])+/))
63
- @rex_tokens.push action { [:HASH, text] }
57
+ when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*/))
58
+ action { [:IDENT, text] }
64
59
 
65
- when (text = ss.scan(/[\s\r\n\f]*~=[\s\r\n\f]*/))
66
- @rex_tokens.push action { [:INCLUDES, text] }
60
+ when (text = @ss.scan(/\#([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])+/))
61
+ action { [:HASH, text] }
67
62
 
68
- when (text = ss.scan(/[\s\r\n\f]*\|=[\s\r\n\f]*/))
69
- @rex_tokens.push action { [:DASHMATCH, text] }
63
+ when (text = @ss.scan(/[\s\r\n\f]*~=[\s\r\n\f]*/))
64
+ action { [:INCLUDES, text] }
70
65
 
71
- when (text = ss.scan(/[\s\r\n\f]*\^=[\s\r\n\f]*/))
72
- @rex_tokens.push action { [:PREFIXMATCH, text] }
66
+ when (text = @ss.scan(/[\s\r\n\f]*\|=[\s\r\n\f]*/))
67
+ action { [:DASHMATCH, text] }
73
68
 
74
- when (text = ss.scan(/[\s\r\n\f]*\$=[\s\r\n\f]*/))
75
- @rex_tokens.push action { [:SUFFIXMATCH, text] }
69
+ when (text = @ss.scan(/[\s\r\n\f]*\^=[\s\r\n\f]*/))
70
+ action { [:PREFIXMATCH, text] }
76
71
 
77
- when (text = ss.scan(/[\s\r\n\f]*\*=[\s\r\n\f]*/))
78
- @rex_tokens.push action { [:SUBSTRINGMATCH, text] }
72
+ when (text = @ss.scan(/[\s\r\n\f]*\$=[\s\r\n\f]*/))
73
+ action { [:SUFFIXMATCH, text] }
79
74
 
80
- when (text = ss.scan(/[\s\r\n\f]*!=[\s\r\n\f]*/))
81
- @rex_tokens.push action { [:NOT_EQUAL, text] }
75
+ when (text = @ss.scan(/[\s\r\n\f]*\*=[\s\r\n\f]*/))
76
+ action { [:SUBSTRINGMATCH, text] }
82
77
 
83
- when (text = ss.scan(/[\s\r\n\f]*=[\s\r\n\f]*/))
84
- @rex_tokens.push action { [:EQUAL, text] }
78
+ when (text = @ss.scan(/[\s\r\n\f]*!=[\s\r\n\f]*/))
79
+ action { [:NOT_EQUAL, text] }
85
80
 
86
- when (text = ss.scan(/[\s\r\n\f]*\)/))
87
- @rex_tokens.push action { [:RPAREN, text] }
81
+ when (text = @ss.scan(/[\s\r\n\f]*=[\s\r\n\f]*/))
82
+ action { [:EQUAL, text] }
88
83
 
89
- when (text = ss.scan(/[\s\r\n\f]*\[[\s\r\n\f]*/))
90
- @rex_tokens.push action { [:LSQUARE, text] }
84
+ when (text = @ss.scan(/[\s\r\n\f]*\)/))
85
+ action { [:RPAREN, text] }
91
86
 
92
- when (text = ss.scan(/[\s\r\n\f]*\]/))
93
- @rex_tokens.push action { [:RSQUARE, text] }
87
+ when (text = @ss.scan(/[\s\r\n\f]*\[[\s\r\n\f]*/))
88
+ action { [:LSQUARE, text] }
94
89
 
95
- when (text = ss.scan(/[\s\r\n\f]*\+[\s\r\n\f]*/))
96
- @rex_tokens.push action { [:PLUS, text] }
90
+ when (text = @ss.scan(/[\s\r\n\f]*\]/))
91
+ action { [:RSQUARE, text] }
97
92
 
98
- when (text = ss.scan(/[\s\r\n\f]*>[\s\r\n\f]*/))
99
- @rex_tokens.push action { [:GREATER, text] }
93
+ when (text = @ss.scan(/[\s\r\n\f]*\+[\s\r\n\f]*/))
94
+ action { [:PLUS, text] }
100
95
 
101
- when (text = ss.scan(/[\s\r\n\f]*,[\s\r\n\f]*/))
102
- @rex_tokens.push action { [:COMMA, text] }
96
+ when (text = @ss.scan(/[\s\r\n\f]*>[\s\r\n\f]*/))
97
+ action { [:GREATER, text] }
103
98
 
104
- when (text = ss.scan(/[\s\r\n\f]*~[\s\r\n\f]*/))
105
- @rex_tokens.push action { [:TILDE, text] }
99
+ when (text = @ss.scan(/[\s\r\n\f]*,[\s\r\n\f]*/))
100
+ action { [:COMMA, text] }
106
101
 
107
- when (text = ss.scan(/\:not\([\s\r\n\f]*/))
108
- @rex_tokens.push action { [:NOT, text] }
102
+ when (text = @ss.scan(/[\s\r\n\f]*~[\s\r\n\f]*/))
103
+ action { [:TILDE, text] }
109
104
 
110
- when (text = ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)/))
111
- @rex_tokens.push action { [:NUMBER, text] }
105
+ when (text = @ss.scan(/\:not\([\s\r\n\f]*/))
106
+ action { [:NOT, text] }
112
107
 
113
- when (text = ss.scan(/[\s\r\n\f]*\/\/[\s\r\n\f]*/))
114
- @rex_tokens.push action { [:DOUBLESLASH, text] }
108
+ when (text = @ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)/))
109
+ action { [:NUMBER, text] }
115
110
 
116
- when (text = ss.scan(/[\s\r\n\f]*\/[\s\r\n\f]*/))
117
- @rex_tokens.push action { [:SLASH, text] }
111
+ when (text = @ss.scan(/[\s\r\n\f]*\/\/[\s\r\n\f]*/))
112
+ action { [:DOUBLESLASH, text] }
118
113
 
119
- when (text = ss.scan(/U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/))
120
- @rex_tokens.push action {[:UNICODE_RANGE, text] }
114
+ when (text = @ss.scan(/[\s\r\n\f]*\/[\s\r\n\f]*/))
115
+ action { [:SLASH, text] }
121
116
 
122
- when (text = ss.scan(/[\s\t\r\n\f]+/))
123
- @rex_tokens.push action { [:S, text] }
117
+ when (text = @ss.scan(/U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/))
118
+ action {[:UNICODE_RANGE, text] }
124
119
 
125
- when (text = ss.scan(/"([^\n\r\f"]|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f']|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*'/))
126
- @rex_tokens.push action { [:STRING, text] }
120
+ when (text = @ss.scan(/[\s\t\r\n\f]+/))
121
+ action { [:S, text] }
127
122
 
128
- when (text = ss.scan(/./))
129
- @rex_tokens.push action { [text, text] }
123
+ when (text = @ss.scan(/"([^\n\r\f"]|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f']|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*'/))
124
+ action { [:STRING, text] }
130
125
 
131
- else
132
- text = ss.string[ss.pos .. -1]
133
- raise ScanError, "can not match: '" + text + "'"
134
- end # if
126
+ when (text = @ss.scan(/./))
127
+ action { [text, text] }
135
128
 
136
129
  else
137
- raise ScanError, "undefined state: '" + state.to_s + "'"
138
- end # case state
139
- end # until ss
140
- end # def scan_evaluate
130
+ text = @ss.string[@ss.pos .. -1]
131
+ raise ScanError, "can not match: '" + text + "'"
132
+ end # if
133
+
134
+ else
135
+ raise ScanError, "undefined state: '" + state.to_s + "'"
136
+ end # case state
137
+ token
138
+ end # def next_token
141
139
 
142
140
  end # class
143
141
  end
@@ -1,11 +1,7 @@
1
1
  module Nokogiri
2
2
  module CSS
3
3
  class Tokenizer < GeneratedTokenizer
4
- ###
5
- # Scan and tokenize +str+
6
- def scan(str)
7
- scan_evaluate(str)
8
- end
4
+ alias :scan :scan_setup
9
5
  end
10
6
  end
11
7
  end
@@ -47,7 +47,7 @@ module Nokogiri
47
47
  end
48
48
  end
49
49
  sub_set
50
- end
50
+ end
51
51
  end
52
52
  end
53
53
  end