oj 3.16.1 → 3.16.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 380409882636ecb1f8e3e54369f85dc11d7e167db741f5aec15bfadad2180601
-  data.tar.gz: 0c60d6f8a5b7d76904b84d378e41632f67c0e3fa2e61aeacfb680cadf15ef9dc
+  metadata.gz: bd443226b6fbe558bd4a01e5e4e2c150149e697c22c1676007f67923b9832682
+  data.tar.gz: 0a46ac536c4fccaeac2bddde9ff8ef8d50343f5eeff3455b36c5bfb762aef073
 SHA512:
-  metadata.gz: 1cbb6d75ab0c495c8d40983970e8fca54aa846eedb4e031368a98874f1db4ca68f160076ac59c230186316dc686923bad86196fa09bf585f501cf11fe814f9bc
-  data.tar.gz: 9d521e53c4e5311eda8abb0f89b12ae2dd3f8007abb02f0b5ff7728311f6d0b64221a493f8df66a7f223dd12c2c2ba6d47b309867f99e2fe206d440d028368a6
+  metadata.gz: 49e28b9151a1c84d9f97adcea6af90acd1141a194858f29e633bf5e2e1965bb5fab783c015e343f569692e6351ad3957059135f05eb50022b79ee799ae0a306a
+  data.tar.gz: 74f69e8e14c39aee7128f8b7dc23fa1fa60a8df383dec844257e67c6e7f34c58d430ac77bc06571e8d52a29f857ee0162cba8ad1e93ac8bc1f7c2ad0260a0002

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # CHANGELOG
 
+## 3.16.4 - 2024-06-08
+
+- Fixed Oj::Parse EOF issue on larger stream input.
+
+## 3.16.3 - 2023-12-11
+
+- Fixed the gemspec to allow earlier versions of the bigdecimal gem.
+
+## 3.16.2 - 2023-12-06
+
+- Fixed documentation formatting.
+
+- Added option to the "usual" parser to raise an error on an empty input string.
+
 ## 3.16.1 - 2023-09-01
 
 - Fixed exception type on number parsing. (thank you @jasonpenny)

data/ext/oj/cache.c

@@ -260,7 +260,8 @@ void cache_set_expunge_rate(Cache c, int rate) {
     c->xrate = (uint8_t)rate;
 }
 
-void cache_free(Cache c) {
+void cache_free(void *data) {
+    Cache    c = (Cache)data;
     uint64_t i;
 
     for (i = 0; i < c->size; i++) {
@@ -276,7 +277,8 @@ void cache_free(Cache c) {
     OJ_FREE(c);
 }
 
-void cache_mark(Cache c) {
+void cache_mark(void *data) {
+    Cache    c = (Cache)data;
     uint64_t i;
 
 #if !HAVE_PTHREAD_MUTEX_INIT

data/ext/oj/cache.h

@@ -10,10 +10,11 @@
 #define CACHE_MAX_KEY 35
 
 struct _cache;
+typedef struct _cache *Cache;
 
 extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking);
-extern void           cache_free(struct _cache *c);
-extern void           cache_mark(struct _cache *c);
+extern void           cache_free(void *data);
+extern void           cache_mark(void *data);
 extern void           cache_set_form(struct _cache *c, VALUE (*form)(const char *str, size_t len));
 extern VALUE          cache_intern(struct _cache *c, const char *key, size_t len);
 extern void           cache_set_expunge_rate(struct _cache *c, int rate);

data/ext/oj/dump.c

@@ -198,7 +198,18 @@ inline static long rails_xss_friendly_size(const uint8_t *str, size_t len) {
 }
 
 inline static size_t rails_friendly_size(const uint8_t *str, size_t len) {
-    return calculate_string_size(str, len, rails_friendly_chars);
+    long    size = 0;
+    size_t  i    = len;
+    uint8_t hi   = 0;
+
+    for (; 0 < i; str++, i--) {
+        size += rails_friendly_chars[*str];
+        hi |= *str & 0x80;
+    }
+    if (0 == hi) {
+        return size - len * (size_t)'0';
+    }
+    return -(size - len * (size_t)'0');
 }
 
 const char *oj_nan_str(VALUE obj, int opt, int mode, bool plus, int *lenp) {
@@ -727,8 +738,11 @@ static void debug_raise(const char *orig, size_t cnt, int line) {
 
 void oj_dump_raw_json(VALUE obj, int depth, Out out) {
     if (oj_string_writer_class == rb_obj_class(obj)) {
-        StrWriter sw  = (StrWriter)DATA_PTR(obj);
-        size_t    len = sw->out.cur - sw->out.buf;
+        StrWriter sw;
+        size_t    len;
+
+        sw  = oj_str_writer_unwrap(obj);
+        len = sw->out.cur - sw->out.buf;
 
         if (0 < len) {
             len--;
@@ -747,8 +761,9 @@ void oj_dump_raw_json(VALUE obj, int depth, Out out) {
 void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
     size_t      size;
     char       *cmap;
-    const char *orig   = str;
-    bool        has_hi = false;
+    const char *orig                  = str;
+    bool        has_hi                = false;
+    bool        do_unicode_validation = false;
 
     switch (out->opts->escape_mode) {
     case NLEsc:
@@ -769,8 +784,9 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         size = xss_friendly_size((uint8_t *)str, cnt);
         break;
     case JXEsc:
-        cmap = hixss_friendly_chars;
-        size = hixss_friendly_size((uint8_t *)str, cnt);
+        cmap                  = hixss_friendly_chars;
+        size                  = hixss_friendly_size((uint8_t *)str, cnt);
+        do_unicode_validation = true;
         break;
     case RailsXEsc: {
         long sz;
@@ -783,12 +799,22 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         } else {
             size = (size_t)sz;
         }
+        do_unicode_validation = true;
         break;
     }
-    case RailsEsc:
+    case RailsEsc: {
+        long sz;
         cmap = rails_friendly_chars;
-        size = rails_friendly_size((uint8_t *)str, cnt);
+        sz   = rails_friendly_size((uint8_t *)str, cnt);
+        if (sz < 0) {
+            has_hi = true;
+            size   = (size_t)-sz;
+        } else {
+            size = (size_t)sz;
+        }
+        do_unicode_validation = true;
         break;
+    }
     case JSONEsc:
     default: cmap = hibit_friendly_chars; size = hibit_friendly_size((uint8_t *)str, cnt);
     }
@@ -819,7 +845,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         for (; str < end; str++) {
             switch (cmap[(uint8_t)*str]) {
             case '1':
-                if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && check_start <= str) {
+                if (do_unicode_validation && check_start <= str) {
                     if (0 != (0x80 & (uint8_t)*str)) {
                         if (0xC0 == (0xC0 & (uint8_t)*str)) {
                             check_start = check_unicode(str, end, orig);
@@ -843,8 +869,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
                 }
                 break;
             case '3':  // Unicode
-                if (0xe2 == (uint8_t)*str && (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) &&
-                    2 <= end - str) {
+                if (0xe2 == (uint8_t)*str && do_unicode_validation && 2 <= end - str) {
                     if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
                         str = dump_unicode(str, end, out, orig);
                     } else {
@@ -863,8 +888,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
                     APPEND_CHARS(out->cur, "\\u00", 4);
                     dump_hex((uint8_t)*str, out);
                 } else {
-                    if (0xe2 == (uint8_t)*str &&
-                        (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 2 <= end - str) {
+                    if (0xe2 == (uint8_t)*str && do_unicode_validation && 2 <= end - str) {
                         if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
                             str = dump_unicode(str, end, out, orig);
                         } else {
@@ -881,8 +905,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         }
         *out->cur++ = '"';
     }
-    if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 0 < str - orig &&
-        0 != (0x80 & *(str - 1))) {
+    if (do_unicode_validation && 0 < str - orig && 0 != (0x80 & *(str - 1))) {
         uint8_t c = (uint8_t) * (str - 1);
         int     i;
         int     scnt = (int)(str - orig);

data/ext/oj/extconf.rb

@@ -29,10 +29,9 @@ dflags = {
 have_func('rb_gc_mark_movable')
 have_func('stpcpy')
 have_func('pthread_mutex_init')
+have_func('getrlimit', 'sys/resource.h')
 have_func('rb_enc_interned_str')
 have_func('rb_ext_ractor_safe', 'ruby.h')
-# rb_hash_bulk_insert is deep down in a header not included in normal build and that seems to fool have_func.
-have_func('rb_hash_bulk_insert', 'ruby.h') unless '2' == version[0] && '6' == version[1]
 
 dflags['OJ_DEBUG'] = true unless ENV['OJ_DEBUG'].nil?
 

data/ext/oj/fast.c

@@ -40,7 +40,7 @@ typedef struct _doc {
     Leaf         *where;                  // points to current location
     Leaf          where_path[MAX_STACK];  // points to head of path
     char         *json;
-    unsigned long size;                   // number of leaves/branches in the doc
+    unsigned long size;  // number of leaves/branches in the doc
     VALUE         self;
     Batch         batches;
     struct _batch batch0;
@@ -573,7 +573,7 @@ static char *read_quoted_value(ParseInfo pi) {
     char *h     = pi->s;  // head
     char *t     = h;      // tail
 
-    h++;                  // skip quote character
+    h++;  // skip quote character
     t++;
     value = h;
     for (; '"' != *h; h++, t++) {
@@ -765,7 +765,7 @@ static VALUE parse_json(VALUE clas, char *json, bool given) {
     pi.s = pi.str;
     doc_init(doc);
     pi.doc = doc;
-#if IS_WINDOWS
+#if IS_WINDOWS || !defined(HAVE_GETRLIMIT)
     // assume a 1M stack and give half to ruby
     pi.stack_min = (void *)((char *)&pi - (512L * 1024L));
 #else
@@ -780,11 +780,10 @@ static VALUE parse_json(VALUE clas, char *json, bool given) {
         }
     }
 #endif
-    doc->json           = json;
-    self                = TypedData_Wrap_Struct(clas, &oj_doc_type, doc);
-    doc->self           = self;
-    DATA_PTR(doc->self) = doc;
-    result              = rb_protect(protect_open_proc, (VALUE)&pi, &ex);
+    doc->json = json;
+    self      = TypedData_Wrap_Struct(clas, &oj_doc_type, doc);
+    doc->self = self;
+    result    = rb_protect(protect_open_proc, (VALUE)&pi, &ex);
     if (given || 0 != ex) {
         DATA_PTR(doc->self) = NULL;
         // TBD is this needed?
@@ -1609,7 +1608,9 @@ static VALUE doc_dump(int argc, VALUE *argv, VALUE self) {
  *   Oj::Doc.open('[1,2,3]') { |doc| doc.size() }  #=> 4
  */
 static VALUE doc_size(VALUE self) {
-    return ULONG2NUM(((Doc)DATA_PTR(self))->size);
+    Doc d;
+    TypedData_Get_Struct(self, struct _doc, &oj_doc_type, d);
+    return ULONG2NUM(d->size);
 }
 
 /* @overload close() => nil

data/ext/oj/intern.c

@@ -85,20 +85,31 @@ static VALUE form_attr(const char *str, size_t len) {
     return (VALUE)rb_intern3(buf, len + 1, oj_utf8_encoding);
 }
 
+static const rb_data_type_t oj_cache_type = {
+    "Oj/cache",
+    {
+        cache_mark,
+        cache_free,
+        NULL,
+    },
+    0,
+    0,
+};
+
 void oj_hash_init(void) {
     VALUE cache_class = rb_define_class_under(Oj, "Cache", rb_cObject);
     rb_undef_alloc_func(cache_class);
 
     struct _cache *str_cache = cache_create(0, form_str, true, true);
-    str_cache_obj            = Data_Wrap_Struct(cache_class, cache_mark, cache_free, str_cache);
+    str_cache_obj            = TypedData_Wrap_Struct(cache_class, &oj_cache_type, str_cache);
     rb_gc_register_address(&str_cache_obj);
 
     struct _cache *sym_cache = cache_create(0, form_sym, true, true);
-    sym_cache_obj            = Data_Wrap_Struct(cache_class, cache_mark, cache_free, sym_cache);
+    sym_cache_obj            = TypedData_Wrap_Struct(cache_class, &oj_cache_type, sym_cache);
     rb_gc_register_address(&sym_cache_obj);
 
     struct _cache *attr_cache = cache_create(0, form_attr, false, true);
-    attr_cache_obj            = Data_Wrap_Struct(cache_class, cache_mark, cache_free, attr_cache);
+    attr_cache_obj            = TypedData_Wrap_Struct(cache_class, &oj_cache_type, attr_cache);
     rb_gc_register_address(&attr_cache_obj);
 
     memset(class_hash.slots, 0, sizeof(class_hash.slots));
@@ -118,17 +129,23 @@ oj_str_intern(const char *key, size_t len) {
 #if HAVE_RB_ENC_INTERNED_STR && 0
     return rb_enc_interned_str(key, len, rb_utf8_encoding());
 #else
-    return cache_intern(DATA_PTR(str_cache_obj), key, len);
+    Cache c;
+    TypedData_Get_Struct(str_cache_obj, struct _cache, &oj_cache_type, c);
+    return cache_intern(c, key, len);
 #endif
 }
 
 VALUE
 oj_sym_intern(const char *key, size_t len) {
-    return cache_intern(DATA_PTR(sym_cache_obj), key, len);
+    Cache c;
+    TypedData_Get_Struct(sym_cache_obj, struct _cache, &oj_cache_type, c);
+    return cache_intern(c, key, len);
 }
 
 ID oj_attr_intern(const char *key, size_t len) {
-    return cache_intern(DATA_PTR(attr_cache_obj), key, len);
+    Cache c;
+    TypedData_Get_Struct(attr_cache_obj, struct _cache, &oj_cache_type, c);
+    return cache_intern(c, key, len);
 }
 
 static uint64_t hash_calc(const uint8_t *key, size_t len) {

data/ext/oj/mimic_json.c

@@ -425,7 +425,7 @@ static VALUE mimic_generate_core(int argc, VALUE *argv, Options copts) {
  *   - *:object_nl* [_String_] String placed after a JSON object
  *   - *:array_nl* [_String_] String placed after a JSON array
  *   - *:ascii_only* [_Boolean_] if not nil or false then use only ascii characters in the output.
- * Note JSON.generate does support this even if it is not documented.
+ *     Note JSON.generate does support this even if it is not documented.
  *
  * Returns [_String_] generated JSON.
  */
@@ -605,9 +605,9 @@ static VALUE mimic_parse_core(int argc, VALUE *argv, VALUE self, bool bang) {
  * - *source* [_String_|IO] source to parse
  * - *opts* [_Hash_] options
  *   - *:symbolize* [Boolean] _names flag indicating JSON object keys should be Symbols instead of
- * Strings
+ *     Strings
  *   - *:create_additions* [Boolean] flag indicating a key matching +create_id+ in a JSON object
- * should trigger the creation of Ruby Object
+ *     should trigger the creation of Ruby Object
  *
  * Returns [Object]
  * @see create_id=

data/ext/oj/object.c

@@ -83,8 +83,9 @@ static int parse_num(const char *str, const char *end, int cnt) {
 
 VALUE
 oj_parse_xml_time(const char *str, int len) {
-    VALUE       args[8];
-    const char *end = str + len;
+    VALUE       args[7];
+    const char *end  = str + len;
+    const char *orig = str;
     int         n;
 
     // year
@@ -144,7 +145,9 @@ oj_parse_xml_time(const char *str, int len) {
         char c = *str++;
 
         if ('.' == c) {
-            long long nsec = 0;
+            unsigned long long       num            = 0;
+            unsigned long long       den            = 1;
+            const unsigned long long last_den_limit = ULLONG_MAX / 10;
 
             for (; str < end; str++) {
                 c = *str;
@@ -152,9 +155,14 @@ oj_parse_xml_time(const char *str, int len) {
                     str++;
                     break;
                 }
-                nsec = nsec * 10 + (c - '0');
+                if (den > last_den_limit) {
+                    // bail to Time.parse if there are more fractional digits than a ULLONG rational can hold
+                    return rb_funcall(rb_cTime, oj_parse_id, 1, rb_str_new(orig, len));
+                }
+                num = num * 10 + (c - '0');
+                den *= 10;
             }
-            args[5] = rb_float_new((double)n + ((double)nsec + 0.5) / 1000000000.0);
+            args[5] = rb_funcall(INT2NUM(n), oj_plus_id, 1, rb_rational_new(ULL2NUM(num), ULL2NUM(den)));
         } else {
             args[5] = rb_ll2inum(n);
         }