oj 3.16.3 → 3.16.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a6b15904386e7010fa071ecc27b8f3a6d5023c8162ea7042fc39e29c373a20a
-  data.tar.gz: b862e59cff0ba3a15eadec34fe807d5893c9c1d1e9fcce01ced66ab28e642181
+  metadata.gz: 6eef89e8f6c362f4ff8c384bba7a892bff9e730a1225bb0bc986acf50ba45dd4
+  data.tar.gz: 2d915ffcec2b55a4a91ad2292053e847efa5aa4ee6ef24e686cadd30218d7533
 SHA512:
-  metadata.gz: f186fb40ebcc3792b4949f81ce3b2efb22bee9dce9b44785eeca37cc27e25b79e8afa4fc5872232001d28a3d0b279fc222d5f7d2f8aaa091fe3a135246139b64
-  data.tar.gz: 9c1536bfc15eb3fe02e92d803417ef1ccce5084036aaf8df5ae2b61995a429ab8ae00be5e9b7c79e11839d781c67523ed5bc534df36a40b59dcc61db6551158f
+  metadata.gz: 845799a9624881305218a82d4d7ceb1ee535da2192ee44e30d38cd4727739749b5c9b18d891e5b363ca0a9b8e8025ee89a66f789f0bf4338493f2c34e1f6a576
+  data.tar.gz: f2cad391b8de831efbf419332b5cc12f29e9cfae1a7a1cf4bf98348de722a3b400cddb4aa29a98576f38cf929061fd57f6e97424eff2bf0e33a9d5f78cfa27c1

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # CHANGELOG
 
+## 3.16.5 - 2024-08-07
+
+- Fixed Oj::Parser so that block procedures work correctly.
+
+## 3.16.4 - 2024-06-08
+
+- Fixed Oj::Parse EOF issue on larger stream input.
+
 ## 3.16.3 - 2023-12-11
 
 - Fixed the gemspec to allow earlier versions of the bigdecimal gem.

data/ext/oj/dump.c

@@ -198,7 +198,18 @@ inline static long rails_xss_friendly_size(const uint8_t *str, size_t len) {
 }
 
 inline static size_t rails_friendly_size(const uint8_t *str, size_t len) {
-    return calculate_string_size(str, len, rails_friendly_chars);
+    long    size = 0;
+    size_t  i    = len;
+    uint8_t hi   = 0;
+
+    for (; 0 < i; str++, i--) {
+        size += rails_friendly_chars[*str];
+        hi |= *str & 0x80;
+    }
+    if (0 == hi) {
+        return size - len * (size_t)'0';
+    }
+    return -(size - len * (size_t)'0');
 }
 
 const char *oj_nan_str(VALUE obj, int opt, int mode, bool plus, int *lenp) {
@@ -750,8 +761,9 @@ void oj_dump_raw_json(VALUE obj, int depth, Out out) {
 void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
     size_t      size;
     char       *cmap;
-    const char *orig   = str;
-    bool        has_hi = false;
+    const char *orig                  = str;
+    bool        has_hi                = false;
+    bool        do_unicode_validation = false;
 
     switch (out->opts->escape_mode) {
     case NLEsc:
@@ -772,8 +784,9 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         size = xss_friendly_size((uint8_t *)str, cnt);
         break;
     case JXEsc:
-        cmap = hixss_friendly_chars;
-        size = hixss_friendly_size((uint8_t *)str, cnt);
+        cmap                  = hixss_friendly_chars;
+        size                  = hixss_friendly_size((uint8_t *)str, cnt);
+        do_unicode_validation = true;
         break;
     case RailsXEsc: {
         long sz;
@@ -786,12 +799,22 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         } else {
             size = (size_t)sz;
         }
+        do_unicode_validation = true;
         break;
     }
-    case RailsEsc:
+    case RailsEsc: {
+        long sz;
         cmap = rails_friendly_chars;
-        size = rails_friendly_size((uint8_t *)str, cnt);
+        sz   = rails_friendly_size((uint8_t *)str, cnt);
+        if (sz < 0) {
+            has_hi = true;
+            size   = (size_t)-sz;
+        } else {
+            size = (size_t)sz;
+        }
+        do_unicode_validation = true;
         break;
+    }
     case JSONEsc:
     default: cmap = hibit_friendly_chars; size = hibit_friendly_size((uint8_t *)str, cnt);
     }
@@ -822,7 +845,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         for (; str < end; str++) {
             switch (cmap[(uint8_t)*str]) {
             case '1':
-                if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && check_start <= str) {
+                if (do_unicode_validation && check_start <= str) {
                     if (0 != (0x80 & (uint8_t)*str)) {
                         if (0xC0 == (0xC0 & (uint8_t)*str)) {
                             check_start = check_unicode(str, end, orig);
@@ -846,8 +869,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
                 }
                 break;
             case '3':  // Unicode
-                if (0xe2 == (uint8_t)*str && (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
-                    2 <= end - str) {
+                if (0xe2 == (uint8_t)*str && do_unicode_validation && 2 <= end - str) {
                     if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
                         str = dump_unicode(str, end, out, orig);
                     } else {
@@ -866,8 +888,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
                     APPEND_CHARS(out->cur, "\\u00", 4);
                     dump_hex((uint8_t)*str, out);
                 } else {
-                    if (0xe2 == (uint8_t)*str &&
-                        (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 2 <= end - str) {
+                    if (0xe2 == (uint8_t)*str && do_unicode_validation && 2 <= end - str) {
                         if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
                             str = dump_unicode(str, end, out, orig);
                         } else {
@@ -884,8 +905,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         }
         *out->cur++ = '"';
     }
-    if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 0 < str - orig &&
-        0 != (0x80 & *(str - 1))) {
+    if (do_unicode_validation && 0 < str - orig && 0 != (0x80 & *(str - 1))) {
         uint8_t c = (uint8_t) * (str - 1);
         int     i;
         int     scnt = (int)(str - orig);

data/ext/oj/extconf.rb

@@ -29,10 +29,9 @@ dflags = {
 have_func('rb_gc_mark_movable')
 have_func('stpcpy')
 have_func('pthread_mutex_init')
+have_func('getrlimit', 'sys/resource.h')
 have_func('rb_enc_interned_str')
 have_func('rb_ext_ractor_safe', 'ruby.h')
-# rb_hash_bulk_insert is deep down in a header not included in normal build and that seems to fool have_func.
-have_func('rb_hash_bulk_insert', 'ruby.h') unless '2' == version[0] && '6' == version[1]
 
 dflags['OJ_DEBUG'] = true unless ENV['OJ_DEBUG'].nil?
 

data/ext/oj/fast.c

@@ -40,7 +40,7 @@ typedef struct _doc {
     Leaf         *where;                  // points to current location
     Leaf          where_path[MAX_STACK];  // points to head of path
     char         *json;
-    unsigned long size;                   // number of leaves/branches in the doc
+    unsigned long size;  // number of leaves/branches in the doc
     VALUE         self;
     Batch         batches;
     struct _batch batch0;
@@ -573,7 +573,7 @@ static char *read_quoted_value(ParseInfo pi) {
     char *h     = pi->s;  // head
     char *t     = h;      // tail
 
-    h++;                  // skip quote character
+    h++;  // skip quote character
     t++;
     value = h;
     for (; '"' != *h; h++, t++) {
@@ -765,7 +765,7 @@ static VALUE parse_json(VALUE clas, char *json, bool given) {
     pi.s = pi.str;
     doc_init(doc);
     pi.doc = doc;
-#if IS_WINDOWS
+#if IS_WINDOWS || !defined(HAVE_GETRLIMIT)
     // assume a 1M stack and give half to ruby
     pi.stack_min = (void *)((char *)&pi - (512L * 1024L));
 #else

data/ext/oj/oj.c

@@ -36,6 +36,7 @@ ID oj_as_json_id;
 ID oj_begin_id;
 ID oj_bigdecimal_id;
 ID oj_end_id;
+ID oj_eofq_id;
 ID oj_exclude_end_id;
 ID oj_error_id;
 ID oj_file_id;
@@ -1849,6 +1850,7 @@ void Init_oj(void) {
     oj_begin_id        = rb_intern("begin");
     oj_bigdecimal_id   = rb_intern("BigDecimal");
     oj_end_id          = rb_intern("end");
+    oj_eofq_id         = rb_intern("eof?");
     oj_error_id        = rb_intern("error");
     oj_exclude_end_id  = rb_intern("exclude_end?");
     oj_file_id         = rb_intern("file?");

data/ext/oj/oj.h

@@ -334,6 +334,7 @@ extern ID oj_as_json_id;
 extern ID oj_begin_id;
 extern ID oj_bigdecimal_id;
 extern ID oj_end_id;
+extern ID oj_eofq_id;
 extern ID oj_error_id;
 extern ID oj_exclude_end_id;
 extern ID oj_file_id;

data/ext/oj/parser.c

@@ -1114,9 +1114,6 @@ static void parse(ojParser p, const byte *json) {
             p->map = trail_map;
         }
     }
-    if (0 < p->depth) {
-        parse_error(p, "parse error, not closed");
-    }
     if (0 == p->depth) {
         switch (p->map[256]) {
         case '0':
@@ -1394,6 +1391,12 @@ static VALUE load(VALUE self) {
         if (0 < RSTRING_LEN(rbuf)) {
             parse(p, (byte *)StringValuePtr(rbuf));
         }
+        if (Qtrue == rb_funcall(p->reader, oj_eofq_id, 0)) {
+            if (0 < p->depth) {
+                parse_error(p, "parse error, not closed");
+            }
+            break;
+        }
     }
     return Qtrue;
 }

data/ext/oj/parser.h

@@ -32,9 +32,9 @@ typedef struct _num {
     long double dub;
     int64_t     fixnum;  // holds all digits
     uint32_t    len;
-    int16_t     div;     // 10^div
+    int16_t     div;  // 10^div
     int16_t     exp;
-    uint8_t     shift;   // shift of fixnum to get decimal
+    uint8_t     shift;  // shift of fixnum to get decimal
     bool        neg;
     bool        exp_neg;
     // for numbers as strings, reuse buf

data/ext/oj/reader.c

@@ -101,7 +101,7 @@ int oj_reader_read(Reader reader) {
         } else {
             shift = reader->pro - reader->head - 1;  // leave one character so we can backup one
         }
-        if (0 >= shift) {                            /* no space left so allocate more */
+        if (0 >= shift) { /* no space left so allocate more */
             const char *old  = reader->head;
             size_t      size = reader->end - reader->head + BUF_PAD;
 

data/ext/oj/saj.c

@@ -578,7 +578,7 @@ static void saj_parse(VALUE handler, char *json) {
     /* initialize parse info */
     pi.str = json;
     pi.s   = json;
-#if IS_WINDOWS
+#if IS_WINDOWS || !defined(HAVE_GETRLIMIT)
     pi.stack_min = (void *)((char *)&obj - (512L * 1024L)); /* assume a 1M stack and give half to ruby */
 #else
     {
@@ -587,7 +587,7 @@ static void saj_parse(VALUE handler, char *json) {
         if (0 == getrlimit(RLIMIT_STACK, &lim) && RLIM_INFINITY != lim.rlim_cur) {
             pi.stack_min = (void *)((char *)&obj - (lim.rlim_cur / 4 * 3)); /* let 3/4ths of the stack be used only */
         } else {
-            pi.stack_min = 0;                                               /* indicates not to check stack limit */
+            pi.stack_min = 0; /* indicates not to check stack limit */
         }
     }
 #endif

data/ext/oj/usual.c

@@ -281,7 +281,6 @@ static void close_object(ojParser p) {
     VALUE         *head = d->vhead + c->vi + 1;
     volatile VALUE obj  = rb_hash_new();
 
-#if HAVE_RB_HASH_BULK_INSERT
     for (vp = head; kp < d->ktail; kp++, vp += 2) {
         *vp = d->get_key(p, kp);
         if (sizeof(kp->buf) <= (size_t)kp->len) {
@@ -289,18 +288,15 @@ static void close_object(ojParser p) {
         }
     }
     rb_hash_bulk_insert(d->vtail - head, head, obj);
-#else
-    for (vp = head; kp < d->ktail; kp++, vp += 2) {
-        rb_hash_aset(obj, d->get_key(p, kp), *(vp + 1));
-        if (sizeof(kp->buf) <= (size_t)kp->len) {
-            OJ_R_FREE(kp->key);
-        }
-    }
-#endif
     d->ktail = d->khead + c->ki;
+
     d->vtail = head;
     head--;
     *head = obj;
+    if (1 == d->vtail - d->vhead && rb_block_given_p()) {
+        d->vtail = d->vhead;
+        rb_yield(obj);
+    }
 }
 
 static void close_object_class(ojParser p) {
@@ -341,7 +337,6 @@ static void close_object_create(ojParser p) {
         head++;
         if (Qnil == d->hash_class) {
             obj = rb_hash_new();
-#if HAVE_RB_HASH_BULK_INSERT
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 *vp = d->get_key(p, kp);
                 if (sizeof(kp->buf) <= (size_t)kp->len) {
@@ -349,14 +344,6 @@ static void close_object_create(ojParser p) {
                 }
             }
             rb_hash_bulk_insert(d->vtail - head, head, obj);
-#else
-            for (vp = head; kp < d->ktail; kp++, vp += 2) {
-                rb_hash_aset(obj, d->get_key(p, kp), *(vp + 1));
-                if (sizeof(kp->buf) <= (size_t)kp->len) {
-                    OJ_R_FREE(kp->key);
-                }
-            }
-#endif
         } else {
             obj = rb_class_new_instance(0, NULL, d->hash_class);
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
@@ -373,7 +360,6 @@ static void close_object_create(ojParser p) {
         if (!d->ignore_json_create && rb_respond_to(clas, oj_json_create_id)) {
             volatile VALUE arg = rb_hash_new();
 
-#if HAVE_RB_HASH_BULK_INSERT
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 *vp = d->get_key(p, kp);
                 if (sizeof(kp->buf) <= (size_t)kp->len) {
@@ -381,14 +367,6 @@ static void close_object_create(ojParser p) {
                 }
             }
             rb_hash_bulk_insert(d->vtail - head, head, arg);
-#else
-            for (vp = head; kp < d->ktail; kp++, vp += 2) {
-                rb_hash_aset(arg, d->get_key(p, kp), *(vp + 1));
-                if (sizeof(kp->buf) <= (size_t)kp->len) {
-                    OJ_R_FREE(kp->key);
-                }
-            }
-#endif
             obj = rb_funcall(clas, oj_json_create_id, 1, arg);
         } else {
             obj = rb_class_new_instance(0, NULL, clas);
@@ -599,7 +577,18 @@ static VALUE result(ojParser p) {
     Usual d = (Usual)p->ctx;
 
     if (d->vhead < d->vtail) {
-        return *d->vhead;
+        long            cnt = d->vtail - d->vhead;
+        volatile VALUE  ary;
+        volatile VALUE *vp;
+
+        if (1 == cnt) {
+            return *d->vhead;
+        }
+        ary = rb_ary_new();
+        for (vp = d->vhead; vp < d->vtail; vp++) {
+            rb_ary_push(ary, *vp);
+        }
+        return ary;
     }
     if (d->raise_on_empty) {
         rb_raise(oj_parse_error_class, "empty string");

data/lib/oj/mimic.rb

@@ -1,11 +1,7 @@
 # frozen_string_literal: false
 
 require 'bigdecimal'
-begin
-  require 'ostruct'
-rescue Exception
-  # ignore
-end
+require 'ostruct'
 
 module Oj
 

data/lib/oj/schandler.rb

@@ -64,13 +64,14 @@ module Oj
   #
   #    hash_end
   #
-  # When a hash key is encountered the hash_key method is called with the parsed
-  # hash value key. The return value from the call is then used as the key in
-  # the key-value pair that follows.
+  #  At the end of a JSON object element the hash_end() callback is called if
+  #  public.
   #
   #    hash_key
   #
-  # At the end of a JSON object element the hash_end() callback is called if public.
+  # When a hash key is encountered the hash_key() method is called with the
+  # parsed hash value key. The return value from the call is then used as the
+  # key in the key-value pair that follows.
   #
   #    hash_set
   #

data/lib/oj/version.rb

@@ -1,4 +1,4 @@
 module Oj
   # Current version of the module.
-  VERSION = '3.16.3'
+  VERSION = '3.16.5'
 end

data/test/activesupport6/encoding_test.rb

@@ -74,42 +74,77 @@ class TestJSONEncoding < ActiveSupport::TestCase
     ActiveSupport.escape_html_entities_in_json = false
   end
 
-  def test_utf8_string_encoded_properly
-    # The original test seems to expect that
-    # ActiveSupport.escape_html_entities_in_json reverts to true even after
-    # being set to false. I haven't been able to figure that out so the value is
-    # set to true, the default, before running the test. This might be wrong but
-    # for now it will have to do.
-    ActiveSupport.escape_html_entities_in_json = true
-    result = ActiveSupport::JSON.encode("€2.99")
-    assert_equal '"€2.99"', result
-    assert_equal(Encoding::UTF_8, result.encoding)
-
-    result = ActiveSupport::JSON.encode("✎☺")
-    assert_equal '"✎☺"', result
-    assert_equal(Encoding::UTF_8, result.encoding)
+  def test_hash_keys_encoding_without_escaping
+    assert_equal "{\"<>\":\"<>\"}", ActiveSupport::JSON.encode("<>" => "<>")
   end
 
-  def test_non_utf8_string_transcodes
-    s = "二".encode("Shift_JIS")
-    result = ActiveSupport::JSON.encode(s)
-    assert_equal '"二"', result
-    assert_equal Encoding::UTF_8, result.encoding
+  module UnicodeTests
+    def test_utf8_string_encoded_properly
+      result = ActiveSupport::JSON.encode("€2.99")
+      assert_equal '"€2.99"', result
+      assert_equal(Encoding::UTF_8, result.encoding)
+
+      result = ActiveSupport::JSON.encode("✎☺")
+      assert_equal '"✎☺"', result
+      assert_equal(Encoding::UTF_8, result.encoding)
+    end
+
+    def test_non_utf8_string_transcodes
+      s = "二".encode("Shift_JIS")
+      result = ActiveSupport::JSON.encode(s)
+      assert_equal '"二"', result
+      assert_equal Encoding::UTF_8, result.encoding
+    end
+
+    def test_wide_utf8_chars
+      w = "𠜎"
+      result = ActiveSupport::JSON.encode(w)
+      assert_equal '"𠜎"', result
+    end
+
+    def test_wide_utf8_roundtrip
+      hash = { string: "𐒑" }
+      json = ActiveSupport::JSON.encode(hash)
+      decoded_hash = ActiveSupport::JSON.decode(json)
+      assert_equal "𐒑", decoded_hash["string"]
+    end
+
+    def test_invalid_encoding_raises
+      s = "\xAE\xFF\x9F"
+      refute s.valid_encoding?
+
+      # n.b. this raises EncodingError, because we didn't call Oj.mimic_JSON in the test setup; but,
+      # if you do that (even indirectly through Oj.optimize_rails), then this raises a
+      # JSON::GeneratorError instead of an EncodingError.
+      assert_raises(EncodingError) do
+        ActiveSupport::JSON.encode([s])
+      end
+    end
   end
 
-  def test_wide_utf8_chars
-    w = "𠜎"
-    result = ActiveSupport::JSON.encode(w)
-    assert_equal '"𠜎"', result
+  module UnicodeTestsWithEscapingOn
+    def setup
+      ActiveSupport.escape_html_entities_in_json = true
+    end
+
+    def teardown
+      ActiveSupport.escape_html_entities_in_json = false
+    end
+
+    include UnicodeTests
   end
 
-  def test_wide_utf8_roundtrip
-    hash = { string: "𐒑" }
-    json = ActiveSupport::JSON.encode(hash)
-    decoded_hash = ActiveSupport::JSON.decode(json)
-    assert_equal "𐒑", decoded_hash["string"]
+  module UnicodeTestsWithEscapingOff
+    def setup
+      ActiveSupport.escape_html_entities_in_json = false
+    end
+
+    include UnicodeTests
   end
 
+  include UnicodeTestsWithEscapingOn
+  include UnicodeTestsWithEscapingOff
+
   def test_hash_key_identifiers_are_always_quoted
     values = { 0 => 0, 1 => 1, :_ => :_, "$" => "$", "a" => "a", :A => :A, :A0 => :A0, "A0B" => "A0B" }
     assert_equal %w( "$" "A" "A0" "A0B" "_" "a" "0" "1" ).sort, object_keys(ActiveSupport::JSON.encode(values))

data/test/activesupport7/encoding_test.rb

@@ -8,9 +8,18 @@ require "active_support/time"
 require_relative "time_zone_test_helpers"
 require_relative "encoding_test_cases"
 
+require 'oj'
+# Sets the ActiveSupport encoder to be Oj and also wraps the setting of globals.
+Oj::Rails.set_encoder()
+Oj::Rails.optimize()
+
 class TestJSONEncoding < ActiveSupport::TestCase
   include TimeZoneTestHelpers
 
+  def test_is_actually_oj
+    assert_equal Oj::Rails::Encoder, ActiveSupport.json_encoder
+  end
+
   def sorted_json(json)
     if json.start_with?("{") && json.end_with?("}")
       "{" + json[1..-2].split(",").sort.join(",") + "}"
@@ -61,36 +70,77 @@ class TestJSONEncoding < ActiveSupport::TestCase
     ActiveSupport.escape_html_entities_in_json = false
   end
 
-  def test_utf8_string_encoded_properly
-    result = ActiveSupport::JSON.encode("€2.99")
-    assert_equal '"€2.99"', result
-    assert_equal(Encoding::UTF_8, result.encoding)
-
-    result = ActiveSupport::JSON.encode("✎☺")
-    assert_equal '"✎☺"', result
-    assert_equal(Encoding::UTF_8, result.encoding)
+  def test_hash_keys_encoding_without_escaping
+    assert_equal "{\"<>\":\"<>\"}", ActiveSupport::JSON.encode("<>" => "<>")
   end
 
-  def test_non_utf8_string_transcodes
-    s = "二".encode("Shift_JIS")
-    result = ActiveSupport::JSON.encode(s)
-    assert_equal '"二"', result
-    assert_equal Encoding::UTF_8, result.encoding
+  module UnicodeTests
+    def test_utf8_string_encoded_properly
+      result = ActiveSupport::JSON.encode("€2.99")
+      assert_equal '"€2.99"', result
+      assert_equal(Encoding::UTF_8, result.encoding)
+
+      result = ActiveSupport::JSON.encode("✎☺")
+      assert_equal '"✎☺"', result
+      assert_equal(Encoding::UTF_8, result.encoding)
+    end
+
+    def test_non_utf8_string_transcodes
+      s = "二".encode("Shift_JIS")
+      result = ActiveSupport::JSON.encode(s)
+      assert_equal '"二"', result
+      assert_equal Encoding::UTF_8, result.encoding
+    end
+
+    def test_wide_utf8_chars
+      w = "𠜎"
+      result = ActiveSupport::JSON.encode(w)
+      assert_equal '"𠜎"', result
+    end
+
+    def test_wide_utf8_roundtrip
+      hash = { string: "𐒑" }
+      json = ActiveSupport::JSON.encode(hash)
+      decoded_hash = ActiveSupport::JSON.decode(json)
+      assert_equal "𐒑", decoded_hash["string"]
+    end
+
+    def test_invalid_encoding_raises
+      s = "\xAE\xFF\x9F"
+      refute s.valid_encoding?
+
+      # n.b. this raises EncodingError, because we didn't call Oj.mimic_JSON in the test setup; but,
+      # if you do that (even indirectly through Oj.optimize_rails), then this raises a
+      # JSON::GeneratorError instead of an EncodingError.
+      assert_raises(EncodingError) do
+        ActiveSupport::JSON.encode([s])
+      end
+    end
   end
 
-  def test_wide_utf8_chars
-    w = "𠜎"
-    result = ActiveSupport::JSON.encode(w)
-    assert_equal '"𠜎"', result
+  module UnicodeTestsWithEscapingOn
+    def setup
+      ActiveSupport.escape_html_entities_in_json = true
+    end
+
+    def teardown
+      ActiveSupport.escape_html_entities_in_json = false
+    end
+
+    include UnicodeTests
   end
 
-  def test_wide_utf8_roundtrip
-    hash = { string: "𐒑" }
-    json = ActiveSupport::JSON.encode(hash)
-    decoded_hash = ActiveSupport::JSON.decode(json)
-    assert_equal "𐒑", decoded_hash["string"]
+  module UnicodeTestsWithEscapingOff
+    def setup
+      ActiveSupport.escape_html_entities_in_json = false
+    end
+
+    include UnicodeTests
   end
 
+  include UnicodeTestsWithEscapingOn
+  include UnicodeTestsWithEscapingOff
+
   def test_hash_key_identifiers_are_always_quoted
     values = { 0 => 0, 1 => 1, :_ => :_, "$" => "$", "a" => "a", :A => :A, :A0 => :A0, "A0B" => "A0B" }
     assert_equal %w( "$" "A" "A0" "A0B" "_" "a" "0" "1" ).sort, object_keys(ActiveSupport::JSON.encode(values))

data/test/foo.rb

@@ -5,10 +5,22 @@ $LOAD_PATH << '.'
 $LOAD_PATH << File.join(__dir__, '../lib')
 $LOAD_PATH << File.join(__dir__, '../ext')
 
-require 'json'
 require 'oj'
-require 'oj/json'
 
-Oj.mimic_JSON
+reader, writer = IO.pipe
 
-JSON.parse("[]")
+thread =
+  Thread.new do
+    5.times do |id|
+      Oj.to_stream(writer, { "id" => id })
+      sleep(1)
+    end
+
+    writer.close
+  end
+
+p = Oj::Parser.new(:usual)
+p.load(reader) { |data| puts "#{Time.now} -- ID: #{data["id"]}" }
+
+reader.close
+thread.join

data/test/test_parser_usual.rb

@@ -114,6 +114,30 @@ class UsualTest < Minitest::Test
     assert_equal(Float, doc.class)
   end
 
+  def test_multi_parse
+    p = Oj::Parser.new(:usual)
+    out = []
+    p.parse('{"a":1}{"b":{"x":2}} {"c":3}') { |j| out.push(j) }
+    assert_equal([{'a'=>1}, {'b'=>{'x'=>2}},{'c'=>3}], out)
+  end
+
+  def test_multi_load
+    p = Oj::Parser.new(:usual)
+    out = []
+    r, w = IO.pipe
+    thread = Thread.new do
+      ['{"a":1}', '{"b":{"x"', ':2}}{"c":', '3}'].each { |seg|
+        w.write(seg)
+        sleep(0.1)
+      }
+      w.close
+    end
+    p.load(r) { |j| out.push(j) }
+    r.close
+    thread.join
+    assert_equal([{'a'=>1}, {'b'=>{'x'=>2}},{'c'=>3}], out)
+  end
+
   def test_omit_null
     p = Oj::Parser.new(:usual)
     p.omit_null = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oj
 version: !ruby/object:Gem::Version
-  version: 3.16.3
+  version: 3.16.5
 platform: ruby
 authors:
 - Peter Ohler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-11 00:00:00.000000000 Z
+date: 2024-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement