oj 3.6.12 → 3.6.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3517ed09e956844a399f8986027f8990949bb7702c9e88a5b4fa4f2fc9d4eb07
-  data.tar.gz: 16292809c8dced3cfd7dd6cfb110d85c88256d74c08586e4c436f92cab8888dc
+  metadata.gz: 1f5fd9fac91126f270714362850b976b7f4cf1102462aab26c2f4f65f457a905
+  data.tar.gz: e2e200e2ec9e76f073206c084c1a7e45592710e0fbb43a5ba4f9593af74ff583
 SHA512:
-  metadata.gz: bc24329317395830e448ec06373ff27458c33612ce6fec0b4a79515c63f43164afb82a093cdbae7a6d6244885f2df16da0fba449c63b283af814769b4ed78147
-  data.tar.gz: 10ff64b0f48d1962c26f213e92456de839b65ab595c2925890a1eabd590160be355674bde197e0d65f4de4e1c9d246c1ce1c12f1adedc3c2aca967b3fd339390
+  metadata.gz: 3b5f0e4bede1f60d010a51e32141c6b864f92abe17fbe12d310e805ce16b8ef67483cf51dee0914f6a29f54556817d91ed2ffa5dccb64cbd8468c68de0d50f83
+  data.tar.gz: 390a4c2953392485757949aee885fb82b3c54b456f659ba672ae7b8da0b45bee484eb3b7c3ee0ef31a01dd33fe0bac8e877d01c91649c197970defe3e7727399

data/ext/oj/dump.c

@@ -230,12 +230,37 @@ dump_hex(uint8_t c, Out out) {
     *out->cur++ = hex_chars[d];
 }
 
+static void
+raise_invalid_unicode(const char *str, int len, int pos) {
+    char	buf[len + 1];
+    char	c;
+    char	code[32];
+    char	*cp = code;
+    int		i;
+    uint8_t	d;
+
+    *cp++ = '[';
+    for (i = pos; i < len && i - pos < 5; i++) {
+	c = str[i];
+	d = (c >> 4) & 0x0F;
+	*cp++ = hex_chars[d];
+	d = c & 0x0F;
+	*cp++ = hex_chars[d];
+	*cp++ = ' ';
+    }
+    cp--;
+    *cp++ = ']';
+    *cp = '\0';
+    strncpy(buf, str, len);
+    rb_raise(oj_json_generator_error_class, "Invalid Unicode %s at %d in '%s'", code, pos, buf);
+}
+
 static const char*
-dump_unicode(const char *str, const char *end, Out out) {
+dump_unicode(const char *str, const char *end, Out out, const char *orig) {
     uint32_t	code = 0;
     uint8_t	b = *(uint8_t*)str;
     int		i, cnt;
-    
+
     if (0xC0 == (0xE0 & b)) {
 	cnt = 1;
 	code = b & 0x0000001F;
@@ -253,13 +278,13 @@ dump_unicode(const char *str, const char *end, Out out) {
 	code = b & 0x00000001;
     } else {
 	cnt = 0;
-	rb_raise(oj_json_generator_error_class, "Invalid Unicode");
+	raise_invalid_unicode(orig, (int)(end - orig), (int)(str - orig));
     }
     str++;
     for (; 0 < cnt; cnt--, str++) {
 	b = *(uint8_t*)str;
 	if (end <= str || 0x80 != (0xC0 & b)) {
-	    rb_raise(oj_json_generator_error_class, "Invalid Unicode");
+	    raise_invalid_unicode(orig, (int)(end - orig), (int)(str - orig));
 	}
 	code = (code << 6) | (b & 0x0000003F);
     }
@@ -284,7 +309,7 @@ dump_unicode(const char *str, const char *end, Out out) {
 }
 
 static const char*
-check_unicode(const char *str, const char *end) {
+check_unicode(const char *str, const char *end, const char *orig) {
     uint8_t	b = *(uint8_t*)str;
     int		cnt;
     
@@ -299,13 +324,13 @@ check_unicode(const char *str, const char *end) {
     } else if (0xFC == (0xFE & b)) {
 	cnt = 5;
     } else {
-	rb_raise(oj_json_generator_error_class, "Invalid Unicode");
+	raise_invalid_unicode(orig, (int)(end - orig), (int)(str - orig));
     }
     str++;
     for (; 0 < cnt; cnt--, str++) {
 	b = *(uint8_t*)str;
 	if (end <= str || 0x80 != (0xC0 & b)) {
-	    rb_raise(oj_json_generator_error_class, "Invalid Unicode");
+	    raise_invalid_unicode(orig, (int)(end - orig), (int)(str - orig));
 	}
     }
     return str;
@@ -783,9 +808,9 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
 		if (JXEsc == out->opts->escape_mode && check_start <= str) {
 		    if (0 != (0x80 & (uint8_t)*str)) {
 			if (0xC0 == (0xC0 & (uint8_t)*str)) {
-			    check_start = check_unicode(str, end);
+			    check_start = check_unicode(str, end, orig);
 			} else {
-			    rb_raise(oj_json_generator_error_class, "Invalid Unicode");
+			    raise_invalid_unicode(orig, (int)(end - orig), (int)(str - orig));
 			}
 		    }
 		}
@@ -806,14 +831,14 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
 	    case '3': // Unicode
 		if (0xe2 == (uint8_t)*str && JXEsc == out->opts->escape_mode && 2 <= end - str) {
 		    if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
-			str = dump_unicode(str, end, out);
+			str = dump_unicode(str, end, out, orig);
 		    } else {
-			check_start = check_unicode(str, end);
+			check_start = check_unicode(str, end, orig);
 			*out->cur++ = *str;
 		    }
 		    break;
 		}
-		str = dump_unicode(str, end, out);
+		str = dump_unicode(str, end, out, orig);
 		break;
 	    case '6': // control characters
 		if (*(uint8_t*)str < 0x80) {
@@ -825,14 +850,14 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
 		} else {
 		    if (0xe2 == (uint8_t)*str && JXEsc == out->opts->escape_mode && 2 <= end - str) {
 			if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
-			    str = dump_unicode(str, end, out);
+			    str = dump_unicode(str, end, out, orig);
 			} else {
-			    check_start = check_unicode(str, end);
+			    check_start = check_unicode(str, end, orig);
 			    *out->cur++ = *str;
 			}
 			break;
 		    }
-		    str = dump_unicode(str, end, out);
+		    str = dump_unicode(str, end, out, orig);
 		}
 		break;
 	    default:

data/ext/oj/parse.c

@@ -971,7 +971,7 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yie
 	if (0 != line) {
 	    VALUE	ec = rb_obj_class(rb_errinfo());
 
-	    if (rb_eArgError != ec) {
+	    if (rb_eArgError != ec && 0 != ec) {
 		err_class = ec;
 	    }
 	}
@@ -1008,6 +1008,7 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yie
 	oj_rxclass_cleanup(&pi->str_rx);
     }
     if (err_has(&pi->err)) {
+	rb_set_errinfo(Qnil);
 	if (Qnil != pi->err_class) {
 	    pi->err.clas = pi->err_class;
 	}

data/ext/oj/sparse.c

@@ -844,7 +844,8 @@ oj_pi_sparse(int argc, VALUE *argv, ParseInfo pi, int fd) {
 	if (0 != line) {
 	    VALUE	ec = rb_obj_class(rb_errinfo());
 
-	    if (rb_eArgError != ec) {
+	    // Sometimes the claass of the error is 0 which seems broken.
+	    if (rb_eArgError != ec && 0 != ec) {
 		err_class = ec;
 	    }
 	}
@@ -876,7 +877,8 @@ oj_pi_sparse(int argc, VALUE *argv, ParseInfo pi, int fd) {
 	close(fd);
     }
     if (err_has(&pi->err)) {
-	if (Qnil != pi->err_class) {
+	rb_set_errinfo(Qnil);
+	if (Qnil != pi->err_class && 0 != pi->err_class) {
 	    pi->err.clas = pi->err_class;
 	}
 	if (CompatMode == pi->options.mode) {

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module. 
-  VERSION = '3.6.12'
+  VERSION = '3.6.13'
 end

data/test/foo.rb

@@ -6,12 +6,65 @@ $: << '../ext'
 
 require 'oj'
 
-10.times {
-  io = StringIO.new
-  w = Oj::StreamWriter.new(io)
-  w.push_array
-  child = {"id"=>"1", "name"=>"foo", "nilElement"=>nil}
-  w.push_value(child)
-  puts io.string
-}
+#Oj.load(StringIO.new(Oj.dump({ records: 1.upto(25).map{|i| { id: i, name: "record_#{i}" }} }, mode: :strict)))
+
+class MyParser
+  attr_accessor :enum
+
+  def initialize
+    @io = StringIO.new
+    @writer = Oj::StreamWriter.new(@io)
+
+    json_string = Oj.dump({ records: 1.upto(25).map{|i| { id: i, name: "record_#{i}" }} }, mode: :strict)
+    @test_json = StringIO.new(json_string)
+
+    @enum = Enumerator.new do |yielder|
+      @yielder = yielder
+      Oj.sc_parse(self, @test_json)
+    end
+  end
+
+  # Stream parsing methods
+  def hash_start
+    @writer.push_object
+  end
+
+  def hash_end
+    @writer.pop unless @io.eof
+  end
+
+  def hash_key(key)
+    @writer.push_key(key)
+  end
+
+  def hash_set(h, key, value)
+    @writer.push_value(value)
+  end
+
+  def array_start
+    @writer.push_array
+  end
+
+  def array_end
+    @writer.pop
+  end
+
+  def array_append(a, value)
+    yield_data
+  end
+
+  def add_value(value);end
+
+  def yield_data
+    @writer.pop_all
+    @yielder << @io.string
+    @io.reopen("")
+    array_start
+  end
+end
+
+MyParser.new.enum.each.with_index do |r, i|
+  break if i == 0
+end
+
 

data/test/test_various.rb

@@ -393,6 +393,12 @@ class Juice < Minitest::Test
     out = Oj.dump(x)
     assert_equal(json, out)
   end
+  def test_dump_invalid_utf8
+    Oj.default_options = { :escape_mode => :ascii }
+    assert_raises(EncodingError) {
+      Oj.dump(["abc\xbe\x1f\x11"], mode: :strict)
+    }
+  end
 
   # Symbol
   def test_symbol_null

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oj
 version: !ruby/object:Gem::Version
-  version: 3.6.12
+  version: 3.6.13
 platform: ruby
 authors:
 - Peter Ohler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-16 00:00:00.000000000 Z
+date: 2018-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -271,7 +271,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: A fast JSON parser and serializer.