ox 1.4.4 → 1.4.5

data/README.md

@@ -26,9 +26,9 @@ A fast XML parser and Object marshaller as a Ruby gem.
 
 ## <a name="release">Release Notes</a>
 
-### Release 1.4.4
+### Release 1.4.5
 
- - Fixed off by 1 error that caused reallocation error for long XML messages.
+ - Fixed encoding issues with all parsers so that attribute and element names can now be encoded with something other than ASCII.
 
 ## <a name="description">Description</a>
 

data/ext/ox/dump.c

@@ -697,7 +697,7 @@ dump_obj(ID aid, VALUE obj, unsigned int depth, Out out) {
                 out->w_end(out, &e);
             }
 #else
-#if (defined JRUBY || defined RBX_Qnil)
+#if (defined JRUBY || defined RUBINIUS)
             VALUE       vars = rb_funcall2(obj, rb_intern("instance_variables"), 0, 0);
 #else
             VALUE       vars = rb_obj_instance_variables(obj);
@@ -856,6 +856,15 @@ dump_gen_doc(VALUE obj, unsigned int depth, Out out) {
     VALUE       attrs = rb_attr_get(obj, attributes_id);
     VALUE       nodes = rb_attr_get(obj, nodes_id);
 
+    if ('\0' == *out->opts->encoding && Qnil != attrs) {
+	VALUE	renc = rb_hash_lookup(attrs, encoding_sym);
+
+        if (Qnil != renc) {
+	    const char  *enc = StringValuePtr(renc);
+
+	    strncpy(out->opts->encoding, enc, sizeof(out->opts->encoding) - 1);
+        }
+    }
     if (Yes == out->opts->with_xml) {
         dump_value(out, "<?xml", 5);
         if (Qnil != attrs) {
@@ -1022,9 +1031,9 @@ dump_obj_to_xml(VALUE obj, Options copts, Out out) {
     }
     out->indent = copts->indent;
     if (ox_document_clas == clas) {
-        dump_gen_doc(obj, -1, out);
+	dump_gen_doc(obj, -1, out);
     } else if (ox_element_clas == clas) {
-        dump_gen_element(obj, 0, out);
+	dump_gen_element(obj, 0, out);
     } else {
         out->w_start = dump_start;
         out->w_end = dump_end;

data/ext/ox/gen_load.c

@@ -105,6 +105,7 @@ create_prolog_doc(PInfo pi, const char *target, Attr attrs) {
     VALUE       doc;
     VALUE       ah;
     VALUE       nodes;
+    VALUE	sym;
 
     if (0 != pi->h) { // top level object
         rb_raise(rb_eSyntaxError, "Prolog must be the first element in an XML document.\n");
@@ -113,7 +114,19 @@ create_prolog_doc(PInfo pi, const char *target, Attr attrs) {
     doc = rb_obj_alloc(ox_document_clas);
     ah = rb_hash_new();
     for (; 0 != attrs->name; attrs++) {
-        rb_hash_aset(ah, ID2SYM(rb_intern(attrs->name)), rb_str_new2(attrs->value));
+#ifdef HAVE_RUBY_ENCODING_H
+	if (0 != pi->encoding) {
+	    VALUE	rstr = rb_str_new2(attrs->name);
+
+	    rb_enc_associate(rstr, pi->encoding);
+	    sym = rb_funcall(rstr, to_sym_id, 0);
+	} else {
+	    sym = ID2SYM(rb_intern(attrs->name));
+	}
+#else
+	sym = ID2SYM(rb_intern(attrs->name));
+#endif
+        rb_hash_aset(ah, sym, rb_str_new2(attrs->value));
 #ifdef HAVE_RUBY_ENCODING_H
         if (0 == strcmp("encoding", attrs->name)) {
             pi->encoding = rb_enc_find(attrs->value);
@@ -266,7 +279,18 @@ add_element(PInfo pi, const char *ename, Attr attrs, int hasChildren) {
             VALUE   *slot;
 
             if (Qundef == (sym = ox_cache_get(symbol_cache, attrs->name, &slot))) {
+#ifdef HAVE_RUBY_ENCODING_H
+		if (0 != pi->encoding) {
+		    VALUE	rstr = rb_str_new2(attrs->name);
+
+		    rb_enc_associate(rstr, pi->encoding);
+		    sym = rb_funcall(rstr, to_sym_id, 0);
+		} else {
+		    sym = ID2SYM(rb_intern(attrs->name));
+		}
+#else
                 sym = ID2SYM(rb_intern(attrs->name));
+#endif
                 *slot = sym;
             }
             s = rb_str_new2(attrs->value);

data/ext/ox/obj_load.c

@@ -49,7 +49,7 @@ static VALUE    parse_xsd_time(const char *text, VALUE clas);
 static VALUE    parse_double_time(const char *text, VALUE clas);
 static VALUE    parse_regexp(const char *text);
 
-static VALUE            get_var_sym_from_attrs(Attr a);
+static VALUE            get_var_sym_from_attrs(Attr a, void *encoding);
 static VALUE            get_obj_from_attrs(Attr a, PInfo pi);
 static VALUE            get_class_from_attrs(Attr a, PInfo pi);
 static VALUE            classname2class(const char *name, PInfo pi);
@@ -78,15 +78,47 @@ ParseCallbacks   ox_obj_callbacks = &_ox_obj_callbacks;
 extern ParseCallbacks   ox_gen_callbacks;
 
 
+inline static VALUE
+str2sym(const char *str, void *encoding) {
+    VALUE	sym;
+    
+#ifdef HAVE_RUBY_ENCODING_H
+    if (0 != encoding) {
+	VALUE	rstr = rb_str_new2(str);
+
+	rb_enc_associate(rstr, (rb_encoding*)encoding);
+	sym = rb_funcall(rstr, to_sym_id, 0);
+    } else {
+	sym = ID2SYM(rb_intern(str));
+    }
+#else
+    sym = ID2SYM(rb_intern(str));
+#endif
+    return sym;
+}
+
 inline static ID
-name2var(const char *name) {
+name2var(const char *name, void *encoding) {
     VALUE       *slot;
     ID          var_id;
 
     if ('0' <= *name && *name <= '9') {
         var_id = INT2NUM(atoi(name));
     } else if (Qundef == (var_id = ox_cache_get(attr_cache, name, &slot))) {
-        var_id = rb_intern(name);
+#ifdef HAVE_RUBY_ENCODING_H
+	if (0 != encoding) {
+	    VALUE	rstr = rb_str_new2(name);
+	    VALUE	sym;
+	    
+	    rb_enc_associate(rstr, (rb_encoding*)encoding);
+	    sym = rb_funcall(rstr, to_sym_id, 0);
+	    var_id = SYM2ID(sym);
+	} else {
+	    var_id = rb_intern(name);
+	}
+#else
+	var_id = rb_intern(name);
+#endif
         *slot = var_id;
     }
     return var_id;
@@ -205,10 +237,10 @@ classname2class(const char *name, PInfo pi) {
 }
 
 static VALUE
-get_var_sym_from_attrs(Attr a) {
+get_var_sym_from_attrs(Attr a, void *encoding) {
     for (; 0 != a->name; a++) {
         if ('a' == *a->name && '\0' == *(a->name + 1)) {
-            return name2var(a->value);
+            return name2var(a->value, encoding);
         }
     }
     return Qundef;
@@ -420,7 +452,7 @@ add_text(PInfo pi, char *text, int closed) {
         VALUE   *slot;
 
         if (Qundef == (sym = ox_cache_get(symbol_cache, text, &slot))) {
-            sym = ID2SYM(rb_intern(text));
+	    sym = str2sym(text, pi->encoding);
             *slot = sym;
         }
         pi->h->obj = sym;
@@ -472,7 +504,7 @@ add_text(PInfo pi, char *text, int closed) {
         }
         from_base64(text, (u_char*)str);
         if (Qundef == (sym = ox_cache_get(symbol_cache, str, &slot))) {
-            sym = ID2SYM(rb_intern(str));
+	    sym = str2sym(str, pi->encoding);
             *slot = sym;
         }
         pi->h->obj = sym;
@@ -548,7 +580,7 @@ add_element(PInfo pi, const char *ename, Attr attrs, int hasChildren) {
     }
     h = pi->h;
     h->type = *ename;
-    h->var = get_var_sym_from_attrs(attrs);
+    h->var = get_var_sym_from_attrs(attrs, pi->encoding);
     switch (h->type) {
     case NilClassCode:
         h->obj = Qnil;

data/ext/ox/ox.c

@@ -57,6 +57,7 @@ ID      end_element_id;
 ID      end_id;
 ID      error_id;
 ID      excl_id;
+ID	fileno_id;
 ID      inspect_id;
 ID      instruct_id;
 ID      keys_id;
@@ -70,6 +71,7 @@ ID      start_element_id;
 ID      text_id;
 ID      to_c_id;
 ID      to_s_id;
+ID	to_sym_id;
 ID      tv_sec_id;
 ID      tv_usec_id;
 ID      value_id;
@@ -97,6 +99,7 @@ VALUE   with_dtd_sym;
 VALUE   with_instruct_sym;
 VALUE   convert_special_sym;
 VALUE   with_xml_sym;
+
 VALUE   empty_string;
 VALUE   zero_fixnum;
 
@@ -645,6 +648,7 @@ void Init_ox() {
     end_id = rb_intern("@end");
     error_id = rb_intern("error");
     excl_id = rb_intern("@excl");
+    fileno_id = rb_intern("fileno");
     inspect_id = rb_intern("inspect");
     instruct_id = rb_intern("instruct");
     keys_id = rb_intern("keys");
@@ -658,6 +662,7 @@ void Init_ox() {
     text_id = rb_intern("text");
     to_c_id = rb_intern("to_c");
     to_s_id = rb_intern("to_s");
+    to_sym_id = rb_intern("to_sym");
     tv_sec_id = rb_intern("tv_sec");
     tv_usec_id = rb_intern("tv_usec");
     value_id = rb_intern("@value");

data/ext/ox/ox.h

@@ -48,7 +48,14 @@ extern "C" {
 #ifdef JRUBY
 #define NO_RSTRUCT 1
 #endif
-#ifdef RBX_Qnil
+
+#if (defined RBX_Qnil && !defined RUBINIUS)
+#define RUBINIUS
+#endif
+
+#ifdef RUBINIUS
+#undef T_RATIONAL
+#undef T_COMPLEX
 #define NO_RSTRUCT 1
 #endif
 
@@ -176,6 +183,8 @@ struct _PInfo {
     CircArray           circ_array;
 #ifdef HAVE_RUBY_ENCODING_H
     rb_encoding         *encoding;
+#else
+    void		*encoding;
 #endif
     unsigned long       id;             /* set for text types when cirs_array is set */
     int                 trace;
@@ -219,6 +228,7 @@ extern ID       end_element_id;
 extern ID       end_id;
 extern ID       error_id;
 extern ID       excl_id;
+extern ID       fileno_id;
 extern ID       inspect_id;
 extern ID       instruct_id;
 extern ID       keys_id;
@@ -232,6 +242,7 @@ extern ID       start_element_id;
 extern ID       text_id;
 extern ID       to_c_id;
 extern ID       to_s_id;
+extern ID       to_sym_id;
 extern ID       tv_sec_id;
 extern ID       tv_usec_id;
 extern ID       value_id;

data/ext/ox/parse.c

@@ -113,9 +113,7 @@ parse(char *xml, ParseCallbacks pcb, char **endp, int trace, Effort effort) {
     pi.pcb = pcb;
     pi.obj = Qnil;
     pi.circ_array = 0;
-#ifdef ENCODING_INLINE_MAX
     pi.encoding = 0;
-#endif
     pi.trace = trace;
     pi.effort = effort;
     while (1) {

data/ext/ox/sax.c

@@ -170,12 +170,23 @@ is_white(char c) {
 }
 
 inline static VALUE
-str2sym(const char *str) {
+str2sym(const char *str, SaxDrive dr) {
     VALUE       *slot;
     VALUE       sym;
 
     if (Qundef == (sym = ox_cache_get(symbol_cache, str, &slot))) {
+#ifdef HAVE_RUBY_ENCODING_H
+        if (0 != dr->encoding) {
+	    VALUE	rstr = rb_str_new2(str);
+
+            rb_enc_associate(rstr, dr->encoding);
+	    sym = rb_funcall(rstr, to_sym_id, 0);
+        } else {
+	    sym = ID2SYM(rb_intern(str));
+	}
+#else
         sym = ID2SYM(rb_intern(str));
+#endif
         *slot = sym;
     }
     return sym;
@@ -208,7 +219,7 @@ sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, int convert) {
     if (rb_respond_to(io, readpartial_id)) {
         VALUE   rfd;
 
-        if (rb_respond_to(io, rb_intern("fileno")) && Qnil != (rfd = rb_funcall(io, rb_intern("fileno"), 0))) {
+        if (rb_respond_to(io, fileno_id) && Qnil != (rfd = rb_funcall(io, fileno_id, 0))) {
             dr->read_func = read_from_fd;
             dr->fd = FIX2INT(rfd);
         } else {
@@ -218,7 +229,7 @@ sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, int convert) {
     } else if (rb_respond_to(io, read_id)) {
         VALUE   rfd;
 
-        if (rb_respond_to(io, rb_intern("fileno")) && Qnil != (rfd = rb_funcall(io, rb_intern("fileno"), 0))) {
+        if (rb_respond_to(io, fileno_id) && Qnil != (rfd = rb_funcall(io, fileno_id, 0))) {
             dr->read_func = read_from_fd;
             dr->fd = FIX2INT(rfd);
         } else {
@@ -565,7 +576,8 @@ read_element(SaxDrive dr) {
     if ('\0' == (c = read_name_token(dr))) {
         return -1;
     }
-    name = str2sym(dr->str);
+    // TBD encode is needed
+    name = str2sym(dr->str, dr);
     if (dr->has_start_element) {
         VALUE       args[1];
 
@@ -669,7 +681,7 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml) {
             is_encoding = 1;
         }
         if (dr->has_attr) {
-            name = str2sym(dr->str);
+            name = str2sym(dr->str, dr);
         }
         if (is_white(c)) {
             c = next_non_white(dr);

data/lib/ox/version.rb

@@ -1,5 +1,5 @@
 
 module Ox
   # Current version of the module. 
-  VERSION = '1.4.4'
+  VERSION = '1.4.5'
 end

data/test/bug3.rb

@@ -3,21 +3,19 @@
 $: << '../lib'
 $: << '../ext'
 
-if __FILE__ == $0
-  if (i = ARGV.index('-I'))
-    x,path = ARGV.slice!(i, 2)
-    $: << path
-  end
-end
-
 require 'ox'
 
-def dump(cnt = 100000)
-  cnt.times do |i|
-    xml = Ox.dump([:inc, 1])
-    #puts xml
-    
+def name_it()
+  begin
+    "x".foo
+  rescue Exception => e
+    #puts e.message
+    xml = Ox.dump(e, effort: :tolerant)
+    puts xml
+    o = Ox.load(xml, mode: :object)
+    puts o.message
+    puts Ox.dump(e)
   end
 end
 
-dump()
+name_it()

data/test/bug4.rb

@@ -1,11 +1,60 @@
 #!/usr/bin/env ruby -wW1
+# encoding: UTF-8
 
-$: << '../lib'
-$: << '../ext'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
 
+require 'stringio'
 require 'ox'
 
-#xml = File.read('bug4.xml')
-xml = File.read('bug4.xml')
+x1 = %(<?xml version="1.0" encoding="ISO-8859-1" ?><tag key="value">Français</tag>).encode("ISO-8859-1")
+# => "<?xml version=\"1.0\" encoding=\"ISO-8859-1\" ?><tag key=\"value\">Fran\xE7ais</tag>" 
+x1.encoding
+# => #<Encoding:ISO-8859-1> 
+
+x2 = %(<?xml version="1.0" encoding="UTF-8" ?><tag key="value">Français</tag>)
+# => "<?xml version=\"1.0\" encoding=\"UTF-8\" ?><tag key=\"value\">Fran\xC3\xA7ais</tag>"
+x2.encoding
+# => #<Encoding:UTF-8> 
+
+class OH < ::Ox::Sax
+  def start_element(name)
+    puts "EL: #{name} (#{name.encoding})"
+  end
+
+  def end_element(name)
+  end
+
+  def attr(key, value)
+    puts "AT: #{key} => #{value} (#{key.encoding} => #{value.encoding})"
+  end
+
+  def text(value)
+    puts "TX: #{value} (#{value.encoding})"
+  end
+end
+
+::Ox.sax_parse OH.new, StringIO.new(x1)
+   # => AT: version => 1.0 (US-ASCII => ASCII-8BIT)
+   #    AT: encoding => ISO-8859-1 (US-ASCII => ISO-8859-1)
+   #    EL: tag (US-ASCII)
+   #    AT: key => value (US-ASCII => ISO-8859-1)
+   #    TX: Fran�ais (ISO-8859-1)
+
+puts
+
+::Ox.sax_parse OH.new, StringIO.new(x2)
+   # => AT: version => 1.0 (US-ASCII => ASCII-8BIT)
+   #    AT: encoding => UTF-8 (US-ASCII => UTF-8)
+   #    EL: tag (US-ASCII)
+   #    AT: key => value (US-ASCII => UTF-8)
+   #    TX: Français (UTF-8)
+
+puts
+x3 = %(<?xml version="1.0" encoding="ISO-8859-1" ?><tag Português="Español">Français</tag>).encode("ISO-8859-1")
+::Ox.sax_parse OH.new, StringIO.new(x3)
+
+puts
+x4 = %(<?xml version="1.0" encoding="UTF-8" ?><tag Português="Español">Français</tag>)
+::Ox.sax_parse OH.new, StringIO.new(x4)
 
-doc = Ox.parse(xml)

data/test/func.rb

@@ -391,6 +391,38 @@ class Func < ::Test::Unit::TestCase
     end
   end
 
+  def test_full_encoding
+    if RUBY_VERSION.start_with?('1.8')
+      assert(true)
+    else
+      xml = %{<?xml version="1.0" encoding="UTF-8"?>
+<いち name="ピーター" つま="まきえ">ピーター</いち>
+}
+      obj = Ox.load(xml)
+      dumped = Ox.dump(obj, :with_xml => true)
+      assert_equal('UTF-8', dumped.encoding.to_s)
+      assert_equal(xml, dumped)
+    end
+  end
+
+  def test_obj_encoding
+    if RUBY_VERSION.start_with?('1.8')
+      assert(true)
+    else
+      orig = Ox.default_options
+      opts = orig.clone
+      opts[:encoding] = 'UTF-8'
+      opts[:with_xml] = true
+      Ox.default_options = opts
+      begin
+        dump_and_load(Bag.new(:@tsuma => :まきえ), false)
+        dump_and_load(Bag.new(:@つま => :まきえ), false)
+      ensure
+        Ox.default_options = orig
+      end
+    end
+  end
+
   def test_instructions
     xml = Ox.dump("test", :with_instructions => true)
     #puts xml

data/test/sax_example.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby -wW1
+
+$: << '../lib'
+$: << '../ext'
+
+require 'stringio'
+require 'ox'
+
+class Sample < ::Ox::Sax
+  def start_element(name); puts "start: #{name}"; end
+  def end_element(name); puts "end: #{name}"; end
+  def attr(name, value); puts "  #{name} => #{value}"; end
+  def text(value); puts "text #{value}"; end
+end
+
+io = StringIO.new(%{
+<top name="sample">
+  <middle name="second">
+    <bottom name="third"/>
+  </middle>
+</top>
+})
+
+handler = Sample.new()
+Ox.sax_parse(handler, io)
+
+# outputs
+# start: top
+#   name => sample
+# start: middle
+#   name => second
+# start: bottom
+#   name => third
+# end: bottom
+# end: middle
+# end: top
+

data/test/sax_test.rb

@@ -447,4 +447,22 @@ encoding = "UTF-8" ?>},
     end
   end
 
+  def test_sax_full_encoding
+    if RUBY_VERSION.start_with?('1.8')
+      assert(true)
+    else
+      parse_compare(%{<?xml version="1.0" encoding="UTF-8"?>
+<いち name="ピーター" つま="まきえ">ピーター</いち>
+},
+                    [[:instruct, "xml"],
+                     [:attr, :version, "1.0"],
+                     [:attr, :encoding, "UTF-8"],
+                     [:start_element, :いち],
+                     [:attr, :name, 'ピーター'],
+                     [:attr, :つま, 'まきえ'],
+                     [:text, 'ピーター'],
+                     [:end_element, :いち]])
+    end
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ox
 version: !ruby/object:Gem::Version
-  version: 1.4.4
+  version: 1.4.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-13 00:00:00.000000000Z
+date: 2012-01-19 00:00:00.000000000Z
 dependencies: []
 description: ! "A fast XML parser and object serializer that uses only standard C
   lib.\n            \nOptimized XML (Ox), as the name implies was written to provide
@@ -51,7 +51,6 @@ files:
 - ext/ox/parse.c
 - ext/ox/sax.c
 - test/bench.rb
-- test/big.rb
 - test/bug1.rb
 - test/bug2.rb
 - test/bug3.rb
@@ -83,6 +82,7 @@ files:
 - test/perf_sax.rb
 - test/perf_write.rb
 - test/sample.rb
+- test/sax_example.rb
 - test/sax_test.rb
 - test/test.rb
 - test/Sample.graffle
@@ -111,9 +111,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: ox
-rubygems_version: 1.8.10
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: A fast XML parser and object serializer.
 test_files: []
-has_rdoc: true

data/test/big.rb

@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby -wW1
-
-$: << '../lib'
-$: << '../ext'
-
-if __FILE__ == $0
-  while (i = ARGV.index('-I'))
-    x,path = ARGV.slice!(i, 2)
-    $: << path
-  end
-end
-
-require 'ox'
-
-def dump(cnt = 10000)
-  h = { }
-  cnt.times do |i|
-    h[i] = [i * 2, "this is #{i}"]
-  end
-  xml = Ox.dump(h)
-  puts "size: #{xml.size}"
-end
-
-dump(200000)