nokogiri 1.2.0 → 1.2.1

data/History.ja.txt

@@ -1,3 +1,10 @@
+=== 1.2.1 / ２００８年２月２３日
+
+* 修正
+
+  * CSS のセレクターのスペースを修正
+  * Ruby 1.9 のStringのエンコードを修正 （角谷さんに感謝！）
+
 === 1.2.0 / ２００８年２月２２日
 
 * 新しい機能

data/History.txt

@@ -1,3 +1,10 @@
+=== 1.2.1 / 2008-02-23
+
+* Bugfixes
+
+  * Fixed a CSS selector space bug
+  * Fixed Ruby 1.9 String Encoding (Thanks 角谷さん！)
+
 === 1.2.0 / 2008-02-22
 
 * New features

data/Manifest.txt

@@ -151,12 +151,16 @@ test/xml/test_builder.rb
 test/xml/test_cdata.rb
 test/xml/test_comment.rb
 test/xml/test_document.rb
+test/xml/test_document_encoding.rb
 test/xml/test_document_fragment.rb
 test/xml/test_dtd.rb
+test/xml/test_dtd_encoding.rb
 test/xml/test_entity_reference.rb
 test/xml/test_node.rb
+test/xml/test_node_encoding.rb
 test/xml/test_node_set.rb
 test/xml/test_processing_instruction.rb
+test/xml/test_reader_encoding.rb
 test/xml/test_text.rb
 test/xml/test_xpath.rb
 vendor/hoe.rb

data/ext/nokogiri/native.c

@@ -25,7 +25,7 @@ void Init_native()
 
   rb_const_set( mNokogiri,
                 rb_intern("LIBXML_VERSION"),
-                rb_str_new2(LIBXML_DOTTED_VERSION)
+                NOKOGIRI_STR_NEW2(LIBXML_DOTTED_VERSION, "UTF-8")
               );
 
   init_xml_document();

data/ext/nokogiri/native.h

@@ -12,6 +12,46 @@
 #include <libxml/HTMLparser.h>
 #include <libxml/HTMLtree.h>
 
+
+#ifndef UNUSED
+# if defined(__GNUC__)
+#  define MAYBE_UNUSED(name) name __attribute__((unused))
+#  define UNUSED(name) MAYBE_UNUSED(UNUSED_ ## name)
+# else
+#  define MAYBE_UNUSED(name) name
+#  define UNUSED(name) name
+# endif
+#endif
+
+#ifdef HAVE_RUBY_ENCODING_H
+
+#include <ruby/encoding.h>
+
+#define NOKOGIRI_STR_NEW2(str, encoding) \
+  ({ \
+    VALUE _string = rb_str_new2((const char *)str); \
+    if(NULL != encoding) \
+      rb_enc_associate_index(_string, rb_enc_find_index(encoding)); \
+    _string; \
+  })
+
+#define NOKOGIRI_STR_NEW(str, len, encoding) \
+  ({ \
+    VALUE _string = rb_str_new((const char *)str, (long)len); \
+    if(NULL != encoding) \
+      rb_enc_associate_index(_string, rb_enc_find_index(encoding)); \
+    _string; \
+  })
+
+#else
+
+#define NOKOGIRI_STR_NEW2(str, doc) \
+  rb_str_new2((const char *)str)
+
+#define NOKOGIRI_STR_NEW(str, len, doc) \
+  rb_str_new((const char *)str, (long)len)
+#endif
+
 #include <xml_io.h>
 #include <xml_document.h>
 #include <html_document.h>

data/ext/nokogiri/xml_document.c

@@ -20,7 +20,7 @@ static VALUE url(VALUE self)
   Data_Get_Struct(self, xmlDoc, doc);
 
   if(doc->URL)
-    return rb_str_new2((const char *)doc->URL);
+    return NOKOGIRI_STR_NEW2(doc->URL, doc->encoding);
 
   return Qnil;
 }
@@ -72,7 +72,7 @@ static VALUE encoding(VALUE self)
   Data_Get_Struct(self, xmlDoc, doc);
 
   if(!doc->encoding) return Qnil;
-  return rb_str_new2((const char *)doc->encoding);
+  return NOKOGIRI_STR_NEW2(doc->encoding, doc->encoding);
 }
 
 /*

data/ext/nokogiri/xml_dtd.c

@@ -8,20 +8,21 @@ static void notation_copier(void *payload, void *data, xmlChar *name)
   xmlNotationPtr c_notation = (xmlNotationPtr)payload;
 
   VALUE notation = rb_funcall(klass, rb_intern("new"), 3,
-      c_notation->name ? rb_str_new2((const char *)c_notation->name) : Qnil,
-      c_notation->PublicID ? rb_str_new2((const char *)c_notation->PublicID) : Qnil,
-      c_notation->SystemID ? rb_str_new2((const char *)c_notation->SystemID) : Qnil);
+      c_notation->name ? NOKOGIRI_STR_NEW2(c_notation->name, "UTF-8") : Qnil,
+      c_notation->PublicID ? NOKOGIRI_STR_NEW2(c_notation->PublicID, "UTF-8") : Qnil,
+      c_notation->SystemID ? NOKOGIRI_STR_NEW2(c_notation->SystemID, "UTF-8") : Qnil);
 
-  rb_hash_aset(hash, rb_str_new2((const char *)name), notation);
+  rb_hash_aset(hash, NOKOGIRI_STR_NEW2(name, "UTF-8"),notation);
 }
 
-static void element_copier(void *payload, void *data, xmlChar *name)
+static void element_copier(void *_payload, void *data, xmlChar *name)
 {
   VALUE hash = (VALUE)data;
+  xmlNodePtr payload = (xmlNodePtr)_payload;
 
-  VALUE element = Nokogiri_wrap_xml_node((xmlNodePtr)payload);
+  VALUE element = Nokogiri_wrap_xml_node(payload);
 
-  rb_hash_aset(hash, rb_str_new2((const char *)name), element);
+  rb_hash_aset(hash, NOKOGIRI_STR_NEW2(name, payload->doc->encoding), element);
 }
 
 /*

data/ext/nokogiri/xml_node.c

@@ -39,7 +39,7 @@ static VALUE encode_special_chars(VALUE self, VALUE string)
       (const xmlChar *)StringValuePtr(string)
   );
 
-  VALUE encoded_str = rb_str_new2((const char *)encoded);
+  VALUE encoded_str = NOKOGIRI_STR_NEW2(encoded, node->doc->encoding);
   xmlFree(encoded);
 
   return encoded_str;
@@ -236,7 +236,8 @@ static VALUE get(VALUE self, VALUE attribute)
 
   if(NULL == propstr) return Qnil;
 
-  rval = rb_str_new2((char *)propstr) ;
+  rval = NOKOGIRI_STR_NEW2(propstr, node->doc->encoding);
+
   xmlFree(propstr);
   return rval ;
 }
@@ -289,8 +290,9 @@ static VALUE namespace(VALUE self)
 {
   xmlNodePtr node ;
   Data_Get_Struct(self, xmlNode, node);
-  if (node->ns && node->ns->prefix)
-    return rb_str_new2((const char *)node->ns->prefix) ;
+  if (node->ns && node->ns->prefix) {
+    return NOKOGIRI_STR_NEW2(node->ns->prefix, node->doc->encoding);
+  }
   return Qnil ;
 }
 
@@ -354,7 +356,7 @@ static VALUE get_content(VALUE self)
 
   xmlChar * content = xmlNodeGetContent(node);
   if(content) {
-    VALUE rval = rb_str_new2((char *)content);
+    VALUE rval = NOKOGIRI_STR_NEW2(content, node->doc->encoding);
     xmlFree(content);
     return rval;
   }
@@ -427,7 +429,8 @@ static VALUE get_name(VALUE self)
 {
   xmlNodePtr node;
   Data_Get_Struct(self, xmlNode, node);
-  if(node->name) return rb_str_new2((const char *)node->name);
+  if(node->name)
+    return NOKOGIRI_STR_NEW2(node->name, node->doc->encoding);
   return Qnil;
 }
 
@@ -441,11 +444,10 @@ static VALUE path(VALUE self)
 {
   xmlNodePtr node;
   xmlChar *path ;
-  VALUE rval ;
   Data_Get_Struct(self, xmlNode, node);
   
   path = xmlGetNodePath(node);
-  rval = rb_str_new2((char *)path);
+  VALUE rval = NOKOGIRI_STR_NEW2(path, node->doc->encoding);
   xmlFree(path);
   return rval ;
 }
@@ -602,14 +604,12 @@ static VALUE dump_html(VALUE self)
   xmlNodePtr node ;
   Data_Get_Struct(self, xmlNode, node);
 
-  VALUE html;
-
   if(node->doc->type == XML_DOCUMENT_NODE)
     return rb_funcall(self, rb_intern("to_xml"), 0);
 
   buf = xmlBufferCreate() ;
   htmlNodeDump(buf, node->doc, node);
-  html = rb_str_new2((char*)buf->content);
+  VALUE html = NOKOGIRI_STR_NEW2(buf->content, node->doc->encoding);
   xmlBufferFree(buf);
   return html ;
 }
@@ -722,7 +722,10 @@ void Nokogiri_xml_node_namespaces(xmlNodePtr node, VALUE attr_hash)
       sprintf(key, "%s", XMLNS_PREFIX);
     }
 
-    rb_hash_aset(attr_hash, rb_str_new2(key), rb_str_new2((const char*)ns->href)) ;
+    rb_hash_aset(attr_hash,
+        NOKOGIRI_STR_NEW2(key, node->doc->encoding),
+        NOKOGIRI_STR_NEW2(ns->href, node->doc->encoding)
+    );
     if (key != buffer) {
       free(key);
     }

data/ext/nokogiri/xml_reader.c

@@ -131,6 +131,16 @@ static VALUE attribute_nodes(VALUE self)
     rb_iv_set(rb_doc, "@decorators", Qnil);
     ptr->doc->_private = (void *)rb_doc;
   }
+  VALUE enc = rb_iv_get(self, "@encoding");
+
+  if(enc != Qnil && NULL == ptr->doc->encoding) {
+    ptr->doc->encoding = calloc((size_t)RSTRING_LEN(enc), sizeof(char));
+    strncpy(
+      (char *)ptr->doc->encoding,
+      StringValuePtr(enc),
+      (size_t)RSTRING_LEN(enc)
+    );
+  }
 
   Nokogiri_xml_node_properties(ptr, attr);
 
@@ -157,7 +167,9 @@ static VALUE attribute_at(VALUE self, VALUE index)
   );
   if(value == NULL) return Qnil;
 
-  VALUE rb_value = rb_str_new2((const char *)value);
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
+  VALUE rb_value = NOKOGIRI_STR_NEW2(value,
+      RTEST(enc) ? StringValuePtr(enc) : NULL);
   xmlFree(value);
   return rb_value;
 }
@@ -193,7 +205,9 @@ static VALUE reader_attribute(VALUE self, VALUE name)
   }
   if(value == NULL) return Qnil;
 
-  VALUE rb_value = rb_str_new2((const char *)value);
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
+  VALUE rb_value = NOKOGIRI_STR_NEW2(value,
+      RTEST(enc) ? StringValuePtr(enc) : NULL);
   xmlFree(value);
   return rb_value;
 }
@@ -230,22 +244,6 @@ static VALUE depth(VALUE self)
   return INT2NUM(depth);
 }
 
-/*
- * call-seq:
- *   encoding
- *
- * Get the encoding for the document
- */
-static VALUE encoding(VALUE self)
-{
-  xmlTextReaderPtr reader;
-  Data_Get_Struct(self, xmlTextReader, reader);
-  const char * encoding = (const char *)xmlTextReaderConstEncoding(reader);
-  if(encoding == NULL) return Qnil;
-
-  return rb_str_new2(encoding);
-}
-
 /*
  * call-seq:
  *   xml_version
@@ -259,7 +257,7 @@ static VALUE xml_version(VALUE self)
   const char * version = (const char *)xmlTextReaderConstXmlVersion(reader);
   if(version == NULL) return Qnil;
 
-  return rb_str_new2(version);
+  return NOKOGIRI_STR_NEW2(version, "UTF-8");
 }
 
 /*
@@ -275,7 +273,9 @@ static VALUE lang(VALUE self)
   const char * lang = (const char *)xmlTextReaderConstXmlLang(reader);
   if(lang == NULL) return Qnil;
 
-  return rb_str_new2(lang);
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
+  return NOKOGIRI_STR_NEW2(lang,
+      RTEST(enc) ? StringValuePtr(enc) : NULL);
 }
 
 /*
@@ -291,7 +291,9 @@ static VALUE value(VALUE self)
   const char * value = (const char *)xmlTextReaderConstValue(reader);
   if(value == NULL) return Qnil;
 
-  return rb_str_new2(value);
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
+  return NOKOGIRI_STR_NEW2(value,
+      RTEST(enc) ? StringValuePtr(enc) : NULL);
 }
 
 /*
@@ -307,7 +309,9 @@ static VALUE prefix(VALUE self)
   const char * prefix = (const char *)xmlTextReaderConstPrefix(reader);
   if(prefix == NULL) return Qnil;
 
-  return rb_str_new2(prefix);
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
+  return NOKOGIRI_STR_NEW2(prefix,
+      RTEST(enc) ? StringValuePtr(enc) : NULL);
 }
 
 /*
@@ -323,7 +327,9 @@ static VALUE namespace_uri(VALUE self)
   const char * uri = (const char *)xmlTextReaderConstNamespaceUri(reader);
   if(uri == NULL) return Qnil;
 
-  return rb_str_new2(uri);
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
+  return NOKOGIRI_STR_NEW2(uri,
+      RTEST(enc) ? StringValuePtr(enc) : NULL);
 }
 
 /*
@@ -339,7 +345,9 @@ static VALUE local_name(VALUE self)
   const char * name = (const char *)xmlTextReaderConstLocalName(reader);
   if(name == NULL) return Qnil;
 
-  return rb_str_new2(name);
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
+  return NOKOGIRI_STR_NEW2(name,
+      RTEST(enc) ? StringValuePtr(enc) : NULL);
 }
 
 /*
@@ -355,7 +363,9 @@ static VALUE name(VALUE self)
   const char * name = (const char *)xmlTextReaderConstName(reader);
   if(name == NULL) return Qnil;
 
-  return rb_str_new2(name);
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
+  return NOKOGIRI_STR_NEW2(name,
+      RTEST(enc) ? StringValuePtr(enc) : NULL);
 }
 
 /*
@@ -437,7 +447,7 @@ static VALUE from_memory(int argc, VALUE *argv, VALUE klass)
   }
 
   VALUE rb_reader = Data_Wrap_Struct(klass, NULL, dealloc, reader);
-  rb_funcall(rb_reader, rb_intern("initialize"), 0);
+  rb_funcall(rb_reader, rb_intern("initialize"), 2, rb_url, encoding);
 
   return rb_reader;
 }
@@ -468,7 +478,6 @@ void init_xml_reader()
   rb_define_method(klass, "value", value, 0);
   rb_define_method(klass, "lang", lang, 0);
   rb_define_method(klass, "xml_version", xml_version, 0);
-  rb_define_method(klass, "encoding", encoding, 0);
   rb_define_method(klass, "depth", depth, 0);
   rb_define_method(klass, "attribute_count", attribute_count, 0);
   rb_define_method(klass, "attribute", reader_attribute, 1);

data/ext/nokogiri/xml_sax_parser.c

@@ -81,11 +81,14 @@ static void start_element(void * ctx, const xmlChar *name, const xmlChar **atts)
   VALUE self = (VALUE)ctx;
   VALUE doc = rb_funcall(self, rb_intern("document"), 0);
   VALUE attributes = rb_ary_new();
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
   const xmlChar * attr;
   int i = 0;
   if(atts) {
     while((attr = atts[i]) != NULL) {
-      rb_funcall(attributes, rb_intern("<<"), 1, rb_str_new2((const char *)attr));
+      rb_funcall(attributes, rb_intern("<<"), 1,
+          NOKOGIRI_STR_NEW2(attr, RTEST(enc) ? StringValuePtr(enc) : NULL)
+      );
       i++;
     }
   }
@@ -93,7 +96,7 @@ static void start_element(void * ctx, const xmlChar *name, const xmlChar **atts)
   rb_funcall( doc,
               rb_intern("start_element"),
               2,
-              rb_str_new2((const char *)name),
+              NOKOGIRI_STR_NEW2(name, RTEST(enc) ? StringValuePtr(enc) : NULL),
               attributes
   );
 }
@@ -101,23 +104,28 @@ static void start_element(void * ctx, const xmlChar *name, const xmlChar **atts)
 static void end_element(void * ctx, const xmlChar *name)
 {
   VALUE self = (VALUE)ctx;
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
   VALUE doc = rb_funcall(self, rb_intern("document"), 0);
-  rb_funcall(doc, rb_intern("end_element"), 1, rb_str_new2((const char *)name));
+  rb_funcall(doc, rb_intern("end_element"), 1,
+      NOKOGIRI_STR_NEW2(name, RTEST(enc) ? StringValuePtr(enc) : NULL)
+  );
 }
 
 static void characters_func(void * ctx, const xmlChar * ch, int len)
 {
   VALUE self = (VALUE)ctx;
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
   VALUE doc = rb_funcall(self, rb_intern("document"), 0);
-  VALUE str = rb_str_new((const char *)ch, (long)len);
+  VALUE str = NOKOGIRI_STR_NEW(ch, len, RTEST(enc) ? StringValuePtr(enc):NULL);
   rb_funcall(doc, rb_intern("characters"), 1, str);
 }
 
 static void comment_func(void * ctx, const xmlChar * value)
 {
   VALUE self = (VALUE)ctx;
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
   VALUE doc = rb_funcall(self, rb_intern("document"), 0);
-  VALUE str = rb_str_new2((const char *)value);
+  VALUE str = NOKOGIRI_STR_NEW2(value, RTEST(enc) ? StringValuePtr(enc):NULL);
   rb_funcall(doc, rb_intern("comment"), 1, str);
 }
 
@@ -162,6 +170,7 @@ static void warning_func(void * ctx, const char *msg, ...)
 {
   VALUE self = (VALUE)ctx;
   VALUE doc = rb_funcall(self, rb_intern("document"), 0);
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
   char * message;
 
   va_list args;
@@ -169,13 +178,16 @@ static void warning_func(void * ctx, const char *msg, ...)
   vasprintf(&message, msg, args);
   va_end(args);
 
-  rb_funcall(doc, rb_intern("warning"), 1, rb_str_new2(message));
+  rb_funcall(doc, rb_intern("warning"), 1,
+      NOKOGIRI_STR_NEW2(message, RTEST(enc) ? StringValuePtr(enc) : NULL)
+  );
   free(message);
 }
 
 static void error_func(void * ctx, const char *msg, ...)
 {
   VALUE self = (VALUE)ctx;
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
   VALUE doc = rb_funcall(self, rb_intern("document"), 0);
   char * message;
 
@@ -184,15 +196,19 @@ static void error_func(void * ctx, const char *msg, ...)
   vasprintf(&message, msg, args);
   va_end(args);
 
-  rb_funcall(doc, rb_intern("error"), 1, rb_str_new2(message));
+  rb_funcall(doc, rb_intern("error"), 1,
+      NOKOGIRI_STR_NEW2(message, RTEST(enc) ? StringValuePtr(enc) : NULL)
+  );
   free(message);
 }
 
 static void cdata_block(void * ctx, const xmlChar * value, int len)
 {
   VALUE self = (VALUE)ctx;
+  VALUE MAYBE_UNUSED(enc) = rb_iv_get(self, "@encoding");
   VALUE doc = rb_funcall(self, rb_intern("document"), 0);
-  VALUE string = rb_str_new((const char *)value, (long)len);
+  VALUE string =
+    NOKOGIRI_STR_NEW(value, len, RTEST(enc) ? StringValuePtr(enc) : NULL);
   rb_funcall(doc, rb_intern("cdata_block"), 1, string);
 }
 

data/ext/nokogiri/xml_syntax_error.c

@@ -45,7 +45,7 @@ static VALUE str3(VALUE self)
   xmlErrorPtr error;
   Data_Get_Struct(self, xmlError, error);
   if(error->str3)
-    return rb_str_new2(error->str3);
+    return NOKOGIRI_STR_NEW2(error->str3, "UTF-8");
   return Qnil;
 }
 
@@ -60,7 +60,7 @@ static VALUE str2(VALUE self)
   xmlErrorPtr error;
   Data_Get_Struct(self, xmlError, error);
   if(error->str2)
-    return rb_str_new2(error->str2);
+    return NOKOGIRI_STR_NEW2(error->str2, "UTF-8");
   return Qnil;
 }
 
@@ -75,7 +75,7 @@ static VALUE str1(VALUE self)
   xmlErrorPtr error;
   Data_Get_Struct(self, xmlError, error);
   if(error->str1)
-    return rb_str_new2(error->str1);
+    return NOKOGIRI_STR_NEW2(error->str1, "UTF-8");
   return Qnil;
 }
 
@@ -103,7 +103,7 @@ static VALUE file(VALUE self)
   xmlErrorPtr error;
   Data_Get_Struct(self, xmlError, error);
   if(error->file)
-    return rb_str_new2(error->file);
+    return NOKOGIRI_STR_NEW2(error->file, "UTF-8");
 
   return Qnil;
 }
@@ -157,7 +157,7 @@ static VALUE message(VALUE self)
 {
   xmlErrorPtr error;
   Data_Get_Struct(self, xmlError, error);
-  return rb_str_new2(error->message);
+  return NOKOGIRI_STR_NEW2(error->message, "UTF-8");
 }
 
 void Nokogiri_error_array_pusher(void * ctx, xmlErrorPtr error)

data/ext/nokogiri/xml_xpath_context.c

@@ -46,7 +46,7 @@ static void ruby_funcall(xmlXPathParserContextPtr ctx, int nargs)
     obj = valuePop(ctx);
     switch(obj->type) {
       case XPATH_STRING:
-        argv[i] = rb_str_new2((char *)obj->stringval);
+        argv[i] = NOKOGIRI_STR_NEW2(obj->stringval, ctx->context->doc->encoding);
         break;
       case XPATH_BOOLEAN:
         argv[i] = obj->boolval == 1 ? Qtrue : Qfalse;
@@ -58,7 +58,9 @@ static void ruby_funcall(xmlXPathParserContextPtr ctx, int nargs)
         argv[i] = Nokogiri_wrap_xml_node_set(obj->nodesetval);
         break;
       default:
-        argv[i] = rb_str_new2((char *)xmlXPathCastToString(obj));
+        argv[i] = NOKOGIRI_STR_NEW2(
+            xmlXPathCastToString(obj), ctx->context->doc->encoding
+        );
     }
     xmlXPathFreeNodeSetList(obj);
   } while(i-- > 0);

data/ext/nokogiri/xslt_stylesheet.c

@@ -46,7 +46,7 @@ static VALUE serialize(VALUE self, VALUE xmlobj)
     Data_Get_Struct(xmlobj, xmlDoc, xml);
     Data_Get_Struct(self, xsltStylesheet, ss);
     xsltSaveResultToString(&doc_ptr, &doc_len, xml, ss);
-    rval = rb_str_new((char*)doc_ptr, doc_len);
+    rval = NOKOGIRI_STR_NEW(doc_ptr, doc_len, xml->encoding);
     xmlFree(doc_ptr);
     return rval ;
 }

data/lib/nokogiri/css/generated_tokenizer.rb

@@ -83,13 +83,13 @@ class GeneratedTokenizer < GeneratedParser
         when (text = ss.scan(/[\s\r\n\f]*=[\s\r\n\f]*/))
            @rex_tokens.push action { [:EQUAL, text] }
 
-        when (text = ss.scan(/[\s\r\n\f]*\)[\s\r\n\f]*/))
+        when (text = ss.scan(/[\s\r\n\f]*\)/))
            @rex_tokens.push action { [:RPAREN, text] }
 
         when (text = ss.scan(/[\s\r\n\f]*\[[\s\r\n\f]*/))
            @rex_tokens.push action { [:LSQUARE, text] }
 
-        when (text = ss.scan(/[\s\r\n\f]*\][\s\r\n\f]*/))
+        when (text = ss.scan(/[\s\r\n\f]*\]/))
            @rex_tokens.push action { [:RSQUARE, text] }
 
         when (text = ss.scan(/[\s\r\n\f]*\+[\s\r\n\f]*/))

data/lib/nokogiri/css/tokenizer.rex

@@ -32,9 +32,9 @@ rule
             {w}\*={w}        { [:SUBSTRINGMATCH, text] }
             {w}!={w}         { [:NOT_EQUAL, text] }
             {w}={w}          { [:EQUAL, text] }
-            {w}\){w}         { [:RPAREN, text] }
+            {w}\)            { [:RPAREN, text] }
             {w}\[{w}         { [:LSQUARE, text] }
-            {w}\]{w}         { [:RSQUARE, text] }
+            {w}\]            { [:RSQUARE, text] }
             {w}\+{w}         { [:PLUS, text] }
             {w}>{w}          { [:GREATER, text] }
             {w},{w}          { [:COMMA, text] }

data/lib/nokogiri/version.rb

@@ -1,4 +1,4 @@
 module Nokogiri
   # The version of Nokogiri you are using
-  VERSION = '1.2.0'
+  VERSION = '1.2.1'
 end

data/lib/nokogiri/xml.rb

@@ -55,6 +55,10 @@ module Nokogiri
     PARSE_NOXINCNODE  = 1 << 15 # do not generate XINCLUDE START/END nodes
 
     class << self
+      def Reader string, url = nil, encoding = nil, options = 0
+        Reader.from_memory(string, url, encoding, options)
+      end
+
       ###
       # Parse an XML document.  See Nokogiri.XML.
       def parse string_or_io, url = nil, encoding = nil, options = 2159

data/lib/nokogiri/xml/reader.rb

@@ -3,9 +3,11 @@ module Nokogiri
     class Reader
       include Enumerable
       attr_accessor :errors
+      attr_reader :encoding
 
-      def initialize
+      def initialize url = nil, encoding = nil
         @errors = []
+        @encoding = encoding
       end
 
       def attributes

data/lib/nokogiri/xml/sax/parser.rb

@@ -2,8 +2,35 @@ module Nokogiri
   module XML
     module SAX
       class Parser
+        ENCODINGS = {
+          'NONE'        => 0, # No char encoding detected
+          'UTF-8'       => 1, # UTF-8
+          'UTF16LE'     => 2, # UTF-16 little endian
+          'UTF16BE'     => 3, # UTF-16 big endian
+          'UCS4LE'      => 4, # UCS-4 little endian
+          'UCS4BE'      => 5, # UCS-4 big endian
+          'EBCDIC'      => 6, # EBCDIC uh!
+          'UCS4-2143'   => 7, # UCS-4 unusual ordering
+          'UCS4-3412'   => 8, # UCS-4 unusual ordering
+          'UCS2'        => 9, # UCS-2
+          'ISO-8859-1'  => 10, # ISO-8859-1 ISO Latin 1
+          'ISO-8859-2'  => 11, # ISO-8859-2 ISO Latin 2
+          'ISO-8859-3'  => 12, # ISO-8859-3
+          'ISO-8859-4'  => 13, # ISO-8859-4
+          'ISO-8859-5'  => 14, # ISO-8859-5
+          'ISO-8859-6'  => 15, # ISO-8859-6
+          'ISO-8859-7'  => 16, # ISO-8859-7
+          'ISO-8859-8'  => 17, # ISO-8859-8
+          'ISO-8859-9'  => 18, # ISO-8859-9
+          'ISO-2022-JP' => 19, # ISO-2022-JP
+          'SHIFT-JIS'   => 20, # Shift_JIS
+          'EUC-JP'      => 21, # EUC-JP
+          'ASCII'       => 22, # pure ASCII
+        }
+
         attr_accessor :document
         def initialize(doc = XML::SAX::Document.new)
+          @encoding = 'ASCII'
           @document = doc
         end
 
@@ -20,8 +47,9 @@ module Nokogiri
 
         ###
         # Parse given +io+
-        def parse_io io, encoding = 0
-          native_parse_io io, encoding
+        def parse_io io, encoding = 'ASCII'
+          @encoding = encoding
+          native_parse_io io, ENCODINGS[@encoding] || ENCODINGS['ASCII']
         end
 
         ###

data/lib/nokogiri/xml/sax/push_parser.rb

@@ -25,8 +25,9 @@ module Nokogiri
       class PushParser
         attr_accessor :document
 
-        def initialize(doc = XML::SAX::Document.new, file_name = nil)
+        def initialize(doc = XML::SAX::Document.new, file_name = nil, encoding = 'ASCII')
           @document = doc
+          @encoding = encoding
           @sax_parser = XML::SAX::Parser.new(doc)
 
           ## Create our push parser context

data/test/css/test_parser.rb

@@ -71,6 +71,12 @@ module Nokogiri
                       @parser.parse("a[@id='Boing']")
       end
 
+      def test_attributes_with_at_and_stuff
+        ## This is non standard CSS
+        assert_xpath  "//a[@id = 'Boing']//div",
+                      @parser.parse("a[@id='Boing'] div")
+      end
+
       def test_not_equal
         ## This is non standard CSS
         assert_xpath  "//a[child::text() != 'Boing']",

data/test/html/test_document.rb

@@ -105,6 +105,20 @@ module Nokogiri
         assert_equal 3, found.length
       end
 
+      def test_find_by_css_with_square_brackets
+        found = @html.css("div[@id='header'] > h1")
+        found = @html.css("div[@id='header'] h1") # this blows up on commit 6fa0f6d329d9dbf1cc21c0ac72f7e627bb4c05fc
+        assert_equal 1, found.length
+      end
+
+      def test_find_with_function
+        found = @html.css("div:awesome() h1", Class.new {
+          def awesome divs
+            [divs.first]
+          end
+        }.new)
+      end
+
       def test_dup_shallow
         found = @html.search('//div/a').first
         dup = found.dup(0)

data/test/test_reader.rb

@@ -136,8 +136,7 @@ class TestReader < Nokogiri::TestCase
     </awesome>
     eoxml
     reader = Nokogiri::XML::Reader.from_memory(string, nil, 'UTF-8')
-    assert_nil reader.encoding
-    assert_equal [nil], reader.map { |x| x.encoding }.uniq
+    assert_equal ['UTF-8'], reader.map { |x| x.encoding }.uniq
   end
 
   def test_xml_version

data/test/xml/sax/test_parser.rb

@@ -22,6 +22,12 @@ module Nokogiri
           assert @parser.document.errors
           assert @parser.document.errors.length > 0
 
+          if RUBY_VERSION =~ /^1\.9/
+            doc.errors.each do |error|
+              assert_equal 'UTF-8', error.message.encoding.name
+            end
+          end
+
           assert_equal doc.errors.length, @parser.document.errors.length
         end
 
@@ -35,9 +41,45 @@ module Nokogiri
 
         def test_parse_io
           File.open(XML_FILE, 'rb') { |f|
-            @parser.parse_io(f)
+            @parser.parse_io(f, 'UTF-8')
           }
           assert(@parser.document.cdata_blocks.length > 0)
+          if RUBY_VERSION =~ /^1\.9/
+            called = false
+            @parser.document.start_elements.flatten.each do |thing|
+              assert_equal 'UTF-8', thing.encoding.name
+              called = true
+            end
+            assert called
+
+            called = false
+            @parser.document.end_elements.flatten.each do |thing|
+              assert_equal 'UTF-8', thing.encoding.name
+              called = true
+            end
+            assert called
+
+            called = false
+            @parser.document.data.each do |thing|
+              assert_equal 'UTF-8', thing.encoding.name
+              called = true
+            end
+            assert called
+
+            called = false
+            @parser.document.comments.flatten.each do |thing|
+              assert_equal 'UTF-8', thing.encoding.name
+              called = true
+            end
+            assert called
+
+            called = false
+            @parser.document.cdata_blocks.flatten.each do |thing|
+              assert_equal 'UTF-8', thing.encoding.name
+              called = true
+            end
+            assert called
+          end
         end
 
         def test_parse_file

data/test/xml/test_document_encoding.rb

@@ -0,0 +1,25 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
+
+module Nokogiri
+  module XML
+   if RUBY_VERSION =~ /^1\.9/
+    class TestDocumentEncoding < Nokogiri::TestCase
+      def setup
+        @xml = Nokogiri::XML(File.read(XML_FILE), XML_FILE, 'UTF-8')
+      end
+
+      def test_url
+        assert_equal @xml.encoding, @xml.url.encoding.name
+      end
+
+      def test_encoding
+        assert_equal @xml.encoding, @xml.encoding.encoding.name
+      end
+
+      def test_dotted_version
+        assert_equal 'UTF-8', Nokogiri::LIBXML_VERSION.encoding.name
+      end
+    end
+   end
+  end
+end

data/test/xml/test_dtd_encoding.rb

@@ -0,0 +1,30 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
+
+module Nokogiri
+  module XML
+   if RUBY_VERSION =~ /^1\.9/
+    class TestDTDEncoding < Nokogiri::TestCase
+      def setup
+        @xml = Nokogiri::XML(File.read(XML_FILE), XML_FILE, 'UTF-8')
+        assert @dtd = @xml.internal_subset
+      end
+
+      def test_entities
+        @dtd.entities.each do |k,v|
+          assert_equal @xml.encoding, k.encoding.name
+        end
+      end
+
+      def test_notations
+        @dtd.notations.each do |k,notation|
+          assert_equal 'UTF-8', k.encoding.name
+          %w{ name public_id system_id }.each do |attribute|
+            v = notation.send(:"#{attribute}") || next
+            assert_equal 'UTF-8', v.encoding.name
+          end
+        end
+      end
+    end
+   end
+  end
+end

data/test/xml/test_node_encoding.rb

@@ -0,0 +1,76 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
+
+module Nokogiri
+  module XML
+    if RUBY_VERSION =~ /^1\.9/
+      class TestNodeEncoding < Nokogiri::TestCase
+        def setup
+          @html = Nokogiri::HTML(File.read(HTML_FILE), HTML_FILE)
+        end
+
+        def test_get_attribute
+          node = @html.css('a').first
+          assert_equal @html.encoding, node['href'].encoding.name
+        end
+
+        def test_encode_special_chars
+          foo = @html.css('a').first.encode_special_chars('foo')
+          assert_equal @html.encoding, foo.encoding.name
+        end
+
+        def test_content
+          node = @html.css('a').first
+          assert_equal @html.encoding, node.content.encoding.name
+        end
+
+        def test_name
+          node = @html.css('a').first
+          assert_equal @html.encoding, node.name.encoding.name
+        end
+
+        def test_path
+          node = @html.css('a').first
+          assert_equal @html.encoding, node.path.encoding.name
+        end
+
+        def test_namespace
+          xml = <<-eoxml
+<root>
+ <car xmlns:part="http://general-motors.com/">
+  <part:tire>Michelin Model XGV</part:tire>
+ </car>
+ <bicycle xmlns:part="http://schwinn.com/">
+  <part:tire>I'm a bicycle tire!</part:tire>
+ </bicycle>
+</root>
+          eoxml
+          doc = Nokogiri::XML(xml, nil, 'UTF-8')
+          assert_equal 'UTF-8', doc.encoding
+          n = doc.xpath('//part:tire', { 'part' => 'http://schwinn.com/' }).first
+          assert n
+          assert_equal doc.encoding, n.namespace.encoding.name
+        end
+
+        def test_namespace_as_hash
+          xml = <<-eoxml
+<root>
+ <car xmlns:part="http://general-motors.com/">
+  <part:tire>Michelin Model XGV</part:tire>
+ </car>
+ <bicycle xmlns:part="http://schwinn.com/">
+  <part:tire>I'm a bicycle tire!</part:tire>
+ </bicycle>
+</root>
+          eoxml
+          doc = Nokogiri::XML(xml, nil, 'UTF-8')
+          assert_equal 'UTF-8', doc.encoding
+          assert n = doc.xpath('//car').first
+          n.namespaces.each do |k,v|
+            assert_equal doc.encoding, v.encoding.name
+            assert_equal doc.encoding, k.encoding.name
+          end
+        end
+      end
+    end
+  end
+end

data/test/xml/test_reader_encoding.rb

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
+
+module Nokogiri
+  module XML
+    if RUBY_VERSION =~ /^1\.9/
+      class TestReaderEncoding < Nokogiri::TestCase
+        def setup
+          @reader = Nokogiri::XML::Reader(
+            File.read(XML_FILE),
+            XML_FILE,
+            'UTF-8'
+          )
+        end
+
+        def test_attribute_at
+          @reader.each do |node|
+            next unless attribute = node.attribute_at(0)
+            assert_equal @reader.encoding, attribute.encoding.name
+          end
+        end
+
+        def test_attributes
+          @reader.each do |node|
+            node.attributes.each do |k,v|
+              assert_equal @reader.encoding, k.encoding.name
+              assert_equal @reader.encoding, v.encoding.name
+            end
+          end
+        end
+
+        def test_attribute
+          xml = <<-eoxml
+    <x xmlns:tenderlove='http://tenderlovemaking.com/'>
+      <tenderlove:foo awesome='true'>snuggles!</tenderlove:foo>
+    </x>
+          eoxml
+          reader = Nokogiri::XML::Reader(xml, nil, 'UTF-8')
+          reader.each do |node|
+            next unless attribute = node.attribute('awesome')
+            assert_equal reader.encoding, attribute.encoding.name
+          end
+        end
+
+        def test_xml_version
+          @reader.each do |node|
+            next unless version = node.xml_version
+            assert_equal @reader.encoding, version.encoding.name
+          end
+        end
+
+        def test_lang
+          xml = <<-eoxml
+    <awesome>
+      <p xml:lang="en">The quick brown fox jumps over the lazy dog.</p>
+      <p xml:lang="ja">日本語が上手です</p>
+    </awesome>
+          eoxml
+
+          reader = Nokogiri::XML::Reader(xml, nil, 'UTF-8')
+          reader.each do |node|
+            next unless lang = node.lang
+            assert_equal reader.encoding, lang.encoding.name
+          end
+        end
+
+        def test_value
+          called = false
+          @reader.each do |node|
+            next unless value = node.value
+            assert_equal @reader.encoding, value.encoding.name
+            called = true
+          end
+          assert called
+        end
+
+        def test_prefix
+          xml = <<-eoxml
+    <x xmlns:edi='http://ecommerce.example.org/schema'>
+      <edi:foo>hello</edi:foo>
+    </x>
+          eoxml
+          reader = Nokogiri::XML::Reader(xml, nil, 'UTF-8')
+          reader.each do |node|
+            next unless prefix = node.prefix
+            assert_equal reader.encoding, prefix.encoding.name
+          end
+        end
+
+        def test_ns_uri
+          xml = <<-eoxml
+    <x xmlns:edi='http://ecommerce.example.org/schema'>
+      <edi:foo>hello</edi:foo>
+    </x>
+          eoxml
+          reader = Nokogiri::XML::Reader(xml, nil, 'UTF-8')
+          reader.each do |node|
+            next unless uri = node.namespace_uri
+            assert_equal reader.encoding, uri.encoding.name
+          end
+        end
+
+        def test_local_name
+          xml = <<-eoxml
+    <x xmlns:edi='http://ecommerce.example.org/schema'>
+      <edi:foo>hello</edi:foo>
+    </x>
+          eoxml
+          reader = Nokogiri::XML::Reader(xml, nil, 'UTF-8')
+          reader.each do |node|
+            next unless lname = node.local_name
+            assert_equal reader.encoding, lname.encoding.name
+          end
+        end
+
+        def test_name
+          @reader.each do |node|
+            next unless name = node.name
+            assert_equal @reader.encoding, name.encoding.name
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: nokogiri
 version: !ruby/object:Gem::Version 
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors: 
 - Aaron Patterson
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-02-22 00:00:00 -08:00
+date: 2009-02-23 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
@@ -182,12 +182,16 @@ files:
 - test/xml/test_cdata.rb
 - test/xml/test_comment.rb
 - test/xml/test_document.rb
+- test/xml/test_document_encoding.rb
 - test/xml/test_document_fragment.rb
 - test/xml/test_dtd.rb
+- test/xml/test_dtd_encoding.rb
 - test/xml/test_entity_reference.rb
 - test/xml/test_node.rb
+- test/xml/test_node_encoding.rb
 - test/xml/test_node_set.rb
 - test/xml/test_processing_instruction.rb
+- test/xml/test_reader_encoding.rb
 - test/xml/test_text.rb
 - test/xml/test_xpath.rb
 - vendor/hoe.rb
@@ -249,11 +253,15 @@ test_files:
 - test/xml/test_cdata.rb
 - test/xml/test_comment.rb
 - test/xml/test_document.rb
+- test/xml/test_document_encoding.rb
 - test/xml/test_document_fragment.rb
 - test/xml/test_dtd.rb
+- test/xml/test_dtd_encoding.rb
 - test/xml/test_entity_reference.rb
 - test/xml/test_node.rb
+- test/xml/test_node_encoding.rb
 - test/xml/test_node_set.rb
 - test/xml/test_processing_instruction.rb
+- test/xml/test_reader_encoding.rb
 - test/xml/test_text.rb
 - test/xml/test_xpath.rb