oj 3.17.3 → 3.17.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e7458dcdf494ef6b1b283ca86d51fba0b3102ecf5cec13f43682878c01708c80
-  data.tar.gz: ab8099b8b275aa5acab45a012bbc96ef0c860e041be2c4ae7d6f7cd331da1755
+  metadata.gz: 8bbb833f16ca15e4591772c707b059e1db37eee8aaf02240cd2f628bf1876f4c
+  data.tar.gz: 8020f6e49607b38fb120a5a6ea4eea9e5ba3e73f35b3556b7defb0202e93e015
 SHA512:
-  metadata.gz: 60d445fd27bbea120359c21d0c1b0ac1d4fbbd678c8fd28efe620706d382a2f0cc434645967c9bac0b6d08d1065fe99dc4b15e2623d98ca0e15a61eb591a993b
-  data.tar.gz: b7736fd3a7b27f98ef3e4df05464000849403ff33a3f82ecd6d8c51b40d0c2753d23ed14c078eb26506d504de991e35ebe68e5703225c80aab6ebbd28b7a80dc
+  metadata.gz: df072519e6d25cc38a04328022741db7bfe3d39a1545337aa6828982eaf233673ee2e04bb697ffb037aa25e0521258881ea96204d8a5d46b0a93b80dc002e94e
+  data.tar.gz: d0683f49f90c751beeb45b2d1ffb39238fd7c0c42e8c2970ed76eeb8ab2f382221c3ef7cdbc7d619ea29c50b7b3bf29559a8ae1e3ec34aa9b56e13a8fb02eac5

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 3.17.4 - 2026-07-14
+
+- Fixed issue in fast.c where rescuing from a panic did not reset the where counter.
+
+- Multiple fixes thanks to @Watson1978.
+
 ## 3.17.3 - 2026-06-04
 
 - Fixed issue in intern.c and fast.c.

data/ext/oj/dump_compat.c

@@ -162,6 +162,7 @@ static void dump_array(VALUE a, int depth, Out out, bool as_ok) {
                     }
                 }
             } else {
+                assure_size(out, size);
                 fill_indent(out, d2);
             }
             oj_dump_compat_val(RARRAY_AREF(a, i), d2, out, true);

data/ext/oj/dump_object.c

@@ -10,6 +10,24 @@ static const char hex_chars[17] = "0123456789abcdef";
 
 static void dump_obj_attrs(VALUE obj, VALUE clas, slot_t id, int depth, Out out);
 
+// An odd class is dumped as {"^O":class,...} and is rebuilt with a create
+// function when loaded so there is no place to put a "^i" circular id on it.
+// Registering it in the circular cache would consume an id that is never
+// written and any later occurrence would then be dumped as a "^r" that refers
+// to nothing. Instead dump odd classes in full each time they are encountered
+// just like Time, Rational, and Complex are.
+static void dump_circular_obj(VALUE obj, VALUE clas, int depth, Out out) {
+    if (NULL != oj_get_odd(clas)) {
+        dump_obj_attrs(obj, clas, 0, depth, out);
+    } else {
+        long id = oj_check_circular(obj, out);
+
+        if (0 <= id) {
+            dump_obj_attrs(obj, clas, id, depth, out);
+        }
+    }
+}
+
 static void dump_time(VALUE obj, Out out) {
     switch (out->opts->time_format) {
     case RubyTime:
@@ -47,11 +65,7 @@ static void dump_data(VALUE obj, int depth, Out out, bool as_ok) {
                 oj_dump_cstr(str, len, 0, 0, out);
             }
         } else {
-            long id = oj_check_circular(obj, out);
-
-            if (0 <= id) {
-                dump_obj_attrs(obj, clas, id, depth, out);
-            }
+            dump_circular_obj(obj, clas, depth, out);
         }
     }
 }
@@ -74,11 +88,7 @@ static void dump_obj(VALUE obj, int depth, Out out, bool as_ok) {
             oj_dump_raw(str, len, out);
         }
     } else {
-        long id = oj_check_circular(obj, out);
-
-        if (0 <= id) {
-            dump_obj_attrs(obj, clas, id, depth, out);
-        }
+        dump_circular_obj(obj, clas, depth, out);
     }
 }
 

data/ext/oj/fast.c

@@ -92,7 +92,14 @@ inline static void next_non_white(ParseInfo pi) {
         case '\f':
         case '\n':
         case '\r': break;
-        case '/': skip_comment(pi); break;
+        case '/':
+            skip_comment(pi);
+            // A comment that is not terminated by a newline ends on the null
+            // terminator. Stop here so the loop does not step past it.
+            if ('\0' == *pi->s) {
+                return;
+            }
+            break;
         default: return;
         }
     }
@@ -210,10 +217,11 @@ static void skip_comment(ParseInfo pi) {
             if ('*' == *pi->s && '/' == *(pi->s + 1)) {
                 pi->s++;
                 return;
-            } else if ('\0' == *pi->s) {
-                raise_error("comment not terminated", pi->str, pi->s);
             }
         }
+        // The loop only ends on the null terminator so the comment was never
+        // closed.
+        raise_error("comment not terminated", pi->str, pi->s);
     } else if ('/' == *pi->s) {
         for (; 1; pi->s++) {
             switch (*pi->s) {
@@ -710,21 +718,23 @@ static void mark_doc(void *ptr) {
 }
 #ifdef HAVE_RB_GC_MARK_MOVABLE
 static void compact_leaf(Leaf leaf) {
-    switch (leaf->value_type) {
-    case COL_VAL:
-        if (NULL != leaf->elements) {
-            Leaf first = leaf->elements->next;
-            Leaf e     = first;
+    if (NULL != leaf) {
+        switch (leaf->value_type) {
+        case COL_VAL:
+            if (NULL != leaf->elements) {
+                Leaf first = leaf->elements->next;
+                Leaf e     = first;
 
-            do {
-                compact_leaf(e);
-                e = e->next;
-            } while (e != first);
-        }
-        break;
-    case RUBY_VAL: leaf->value = rb_gc_location(leaf->value); break;
+                do {
+                    compact_leaf(e);
+                    e = e->next;
+                } while (e != first);
+            }
+            break;
+        case RUBY_VAL: leaf->value = rb_gc_location(leaf->value); break;
 
-    default: break;
+        default: break;
+        }
     }
 }
 
@@ -790,14 +800,11 @@ static VALUE parse_json(VALUE clas, char *json, bool given) {
     doc->self = self;
     result    = rb_protect(protect_open_proc, (VALUE)&pi, &ex);
     if (given || 0 != ex) {
+        // The doc is detached from its wrapper so the GC will not free it.
+        // doc_free() releases both the doc and its json buffer (doc->json), so
+        // the caller must not free json separately.
         DATA_PTR(doc->self) = NULL;
-        // TBD is this needed?
-        /*
         doc_free(pi.doc);
-        if (0 != ex) {  // will jump so caller will not free
-            OJ_R_FREE(json);
-        }
-        */
     } else {
         result = doc->self;
     }
@@ -946,6 +953,7 @@ static void each_leaf(Doc doc, VALUE self) {
 
             doc->where++;
             if (MAX_STACK <= doc->where - doc->where_path) {
+                doc->where--;
                 rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")), "Path too deep. Limit is %d levels.", MAX_STACK);
             }
             do {
@@ -972,8 +980,9 @@ static int move_step(Doc doc, const char *path, int loc) {
     } else {
         Leaf leaf;
 
+        // A document with no root (empty or comment only JSON) has no leaf to
+        // step from so the path can not be located.
         if (0 == doc->where || 0 == (leaf = *doc->where)) {
-            printf("*** Internal error at %s\n", path);
             return loc;
         }
         if ('.' == *path && '.' == *(path + 1)) {
@@ -1104,12 +1113,7 @@ static VALUE doc_open(VALUE clas, VALUE str) {
 
     memcpy(json, StringValuePtr(str), len);
     obj = parse_json(clas, json, given);
-    // TBD is this needed
-    /*
-    if (given) {
-        OJ_R_FREE(json);
-    }
-    */
+    // json is owned by the doc and freed by doc_free(); do not free it here.
     return obj;
 }
 
@@ -1141,7 +1145,9 @@ static VALUE doc_open_file(VALUE clas, VALUE filename) {
     int            given = rb_block_given_p();
 
     path = StringValuePtr(filename);
-    if (0 == (f = fopen(path, "r"))) {
+    // Open in binary mode. On Windows a text mode read translates CRLF into LF
+    // so fewer bytes are read than the file size reported by ftell().
+    if (0 == (f = fopen(path, "rb"))) {
         rb_raise(rb_eIOError, "%s", strerror(errno));
     }
     fseek(f, 0, SEEK_END);
@@ -1159,12 +1165,7 @@ static VALUE doc_open_file(VALUE clas, VALUE filename) {
     fclose(f);
     json[len] = '\0';
     obj       = parse_json(clas, json, given);
-    // TBD is this needed
-    /*
-    if (given) {
-        OJ_R_FREE(json);
-    }
-    */
+    // json is owned by the doc and freed by doc_free(); do not free it here.
     return obj;
 }
 
@@ -1263,10 +1264,14 @@ static VALUE doc_path(VALUE self) {
  * #=> nil
  */
 static VALUE doc_local_key(VALUE self) {
-    Doc            doc  = self_doc(self);
-    Leaf           leaf = *doc->where;
-    volatile VALUE key  = Qnil;
+    Doc            doc = self_doc(self);
+    Leaf           leaf;
+    volatile VALUE key = Qnil;
 
+    if (NULL == doc->where || NULL == *doc->where) {
+        return Qnil;
+    }
+    leaf = *doc->where;
     if (T_HASH == leaf->parent_type) {
         key = rb_utf8_str_new_cstr(leaf->key);
     } else if (T_ARRAY == leaf->parent_type) {
@@ -1421,6 +1426,9 @@ static VALUE doc_each_leaf(int argc, VALUE *argv, VALUE self) {
                 return Qnil;
             }
         }
+        if (NULL == doc->where || NULL == *doc->where) {
+            return Qnil;
+        }
         each_leaf(doc, self);
         if (0 < wlen) {
             memcpy(doc->where_path, save_path, sizeof(Leaf) * (wlen + 1));

data/ext/oj/object.c

@@ -319,20 +319,26 @@ static int hat_value(ParseInfo pi, Val parent, const char *key, size_t klen, vol
             e1 = *RARRAY_CONST_PTR(value);
             // check for anonymous Struct
             if (T_ARRAY == rb_type(e1)) {
-                VALUE          args[1024];
-                volatile VALUE rstr;
+                // Build the member-name list in a GC-managed Ruby array. Using a
+                // fixed C buffer (VALUE args[1024]) here overflowed the stack when
+                // the JSON supplied more than 1024 member names.
+                volatile VALUE names = rb_ary_new2(RARRAY_LEN(e1));
                 size_t         i;
                 size_t         cnt = RARRAY_LEN(e1);
 
                 for (i = 0; i < cnt; i++) {
-                    rstr    = RARRAY_AREF(e1, i);
-                    args[i] = rb_funcall(rstr, oj_to_sym_id, 0);
+                    rb_ary_push(names, rb_funcall(RARRAY_AREF(e1, i), oj_to_sym_id, 0));
                 }
-                sc = rb_funcall2(rb_cStruct, oj_new_id, (int)cnt, args);
+                sc = rb_apply(rb_cStruct, oj_new_id, names);
             } else {
                 // If struct is not defined then we let this fail and raise an exception.
                 sc = oj_name2struct(pi, *RARRAY_CONST_PTR(value), rb_eArgError);
             }
+            if (Qundef == sc || Qnil == sc) {
+                // oj_name2struct already recorded the error; do not pass an
+                // unresolved (Qundef) class on to rb_obj_alloc/rb_class_new_instance.
+                return 1;
+            }
             if (sc == rb_cRange) {
                 parent->val = rb_class_new_instance((int)(len - 1), RARRAY_CONST_PTR(value) + 1, rb_cRange);
             } else {

data/ext/oj/oj.c

@@ -606,9 +606,52 @@ static VALUE get_def_opts(VALUE self) {
  *   - *:trace* [_Boolean_] turn trace on or off.
  *   - *:safe* [_Boolean_] turn safe mimic on or off.
  */
+static ID oj_default_options_keeper_id = 0;
+
+// The default options are a plain C global so the Ruby objects in them, the
+// classes of the hash_class, array_class, and ignore options and the regexps
+// and the classes of the match_string option, can not be reached by the GC and
+// are collected while the defaults still refer to them. Holding them in an
+// array the GC does know about keeps them alive. The array is rebuilt whenever
+// the defaults change and replaces the previous one so that only the objects
+// the defaults are currently using are held.
+static void keep_default_options(void) {
+    VALUE keep = rb_ary_new();
+
+    if (0 == oj_default_options_keeper_id) {
+        oj_default_options_keeper_id = rb_intern("@default_options_keeper");
+    }
+    if (Qnil != oj_default_options.hash_class) {
+        rb_ary_push(keep, oj_default_options.hash_class);
+    }
+    if (Qnil != oj_default_options.array_class) {
+        rb_ary_push(keep, oj_default_options.array_class);
+    }
+    if (NULL != oj_default_options.ignore) {
+        VALUE *vp;
+
+        for (vp = oj_default_options.ignore; Qnil != *vp; vp++) {
+            rb_ary_push(keep, *vp);
+        }
+    }
+    oj_rxclass_keep(&oj_default_options.str_rx, keep);
+    // An instance variable of the Oj module is a reference the GC can follow on
+    // every engine, unlike a C global.
+    rb_ivar_set(Oj, oj_default_options_keeper_id, keep);
+}
+
 static VALUE set_def_opts(VALUE self, VALUE opts) {
     Check_Type(opts, T_HASH);
+    // A :match_string option replaces the regexps instead of adding to them so
+    // the ones the defaults are holding have to be freed here, the only place
+    // that owns them. A per call options struct only aliases them.
+    if (Qnil != rb_hash_lookup(opts, match_string_sym)) {
+        oj_rxclass_cleanup(&oj_default_options.str_rx);
+        oj_default_options.str_rx.head = NULL;
+        oj_default_options.str_rx.tail = NULL;
+    }
     oj_parse_options(opts, &oj_default_options);
+    keep_default_options();
 
     return Qnil;
 }
@@ -921,6 +964,9 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
 
             len = RSTRING_LEN(v);
             if (len != copts->create_id_len || 0 != strcmp(copts->create_id, str)) {
+                if (&oj_default_options == copts && oj_json_class != copts->create_id) {
+                    OJ_R_FREE((char *)copts->create_id);
+                }
                 copts->create_id = OJ_R_ALLOC_N(char, len + 1);
                 strcpy((char *)copts->create_id, str);
                 copts->create_id_len = len;
@@ -1052,7 +1098,11 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
             copts->array_class = v;
         }
     } else if (ignore_sym == k) {
-        OJ_R_FREE(copts->ignore);
+        // Only free when replacing the defaults; a per-call copy's ignore still
+        // aliases the default-owned buffer, which oj_free_call_options() frees.
+        if (&oj_default_options == copts) {
+            OJ_R_FREE(copts->ignore);
+        }
         copts->ignore = NULL;
         if (Qnil != v) {
             size_t cnt;
@@ -1125,15 +1175,15 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         strncpy(copts->float_fmt, RSTRING_PTR(v), (size_t)RSTRING_LEN(v));
         copts->float_fmt[RSTRING_LEN(v)] = '\0';
     } else if (only_sym == k) {
-        if (NULL != copts->dump_opts.only) {
+        // Only free when replacing the defaults; a per-call copy's pointer still
+        // aliases the default-owned buffer, which oj_free_call_options() frees.
+        if (&oj_default_options == copts && NULL != copts->dump_opts.only) {
             OJ_R_FREE((void *)copts->dump_opts.only);
-            copts->dump_opts.only = NULL;
         }
         copts->dump_opts.only = make_only_value(v);
     } else if (except_sym == k) {
-        if (NULL != copts->dump_opts.except) {
+        if (&oj_default_options == copts && NULL != copts->dump_opts.except) {
             OJ_R_FREE((void *)copts->dump_opts.except);
-            copts->dump_opts.except = NULL;
         }
         copts->dump_opts.except = make_only_value(v);
     }
@@ -1153,6 +1203,120 @@ void oj_parse_options(VALUE ropts, Options copts) {
     return;
 }
 
+// Free the option buffers that oj_parse_options() allocated for a single call.
+// A per-call options struct is a copy of oj_default_options, so its create_id,
+// ignore, only, and except members initially alias the ones owned by the
+// defaults. Only free a member when it differs from the default, i.e. when it
+// was allocated for this call; the default-owned buffers must be left alone.
+void oj_free_call_options(Options copts) {
+    if (NULL != copts->create_id && oj_default_options.create_id != copts->create_id) {
+        OJ_R_FREE((char *)copts->create_id);
+        copts->create_id     = NULL;
+        copts->create_id_len = 0;
+    }
+    if (NULL != copts->ignore && oj_default_options.ignore != copts->ignore) {
+        OJ_R_FREE(copts->ignore);
+        copts->ignore = NULL;
+    }
+    if (NULL != copts->dump_opts.only && oj_default_options.dump_opts.only != copts->dump_opts.only) {
+        OJ_R_FREE((void *)copts->dump_opts.only);
+        copts->dump_opts.only = NULL;
+    }
+    if (NULL != copts->dump_opts.except && oj_default_options.dump_opts.except != copts->dump_opts.except) {
+        OJ_R_FREE((void *)copts->dump_opts.except);
+        copts->dump_opts.except = NULL;
+    }
+}
+
+static char *dup_option_str(const char *str, size_t len) {
+    char *buf = OJ_R_ALLOC_N(char, len + 1);
+
+    memcpy(buf, str, len);
+    buf[len] = '\0';
+
+    return buf;
+}
+
+// Give an options struct private copies of the option buffers it still shares
+// with the defaults so that it owns all of them.
+//
+// An options struct that outlives the call it was created in can not rely on
+// the default-owned buffers it aliases: Oj.default_options= is free to replace
+// them at any time, which would leave the alias dangling. Call this once, in
+// the call that creates such an object and after oj_parse_options(), then free
+// the buffers with oj_options_release() when the object is collected.
+void oj_options_take_ownership(Options copts) {
+    if (NULL != copts->create_id && oj_default_options.create_id == copts->create_id) {
+        copts->create_id = dup_option_str(copts->create_id, copts->create_id_len);
+    }
+    if (NULL != copts->ignore && oj_default_options.ignore == copts->ignore) {
+        VALUE *vp;
+        VALUE *buf;
+        size_t cnt = 0;
+
+        for (vp = copts->ignore; Qnil != *vp; vp++) {
+            cnt++;
+        }
+        buf = OJ_R_ALLOC_N(VALUE, cnt + 1);
+        memcpy(buf, copts->ignore, sizeof(VALUE) * (cnt + 1));
+        copts->ignore = buf;
+    }
+    if (NULL != copts->dump_opts.only && oj_default_options.dump_opts.only == copts->dump_opts.only) {
+        copts->dump_opts.only = dup_option_str(copts->dump_opts.only, strlen(copts->dump_opts.only));
+    }
+    if (NULL != copts->dump_opts.except && oj_default_options.dump_opts.except == copts->dump_opts.except) {
+        copts->dump_opts.except = dup_option_str(copts->dump_opts.except, strlen(copts->dump_opts.except));
+    }
+}
+
+// Free the option buffers of an options struct that owns all of them, i.e. one
+// that oj_options_take_ownership() was called on. Unlike oj_free_call_options()
+// this never looks at the defaults, so it is safe to call from a GC free
+// function, where the defaults may have been replaced since the object was
+// created.
+void oj_options_release(Options copts) {
+    if (NULL != copts->create_id) {
+        OJ_R_FREE((char *)copts->create_id);
+        copts->create_id     = NULL;
+        copts->create_id_len = 0;
+    }
+    if (NULL != copts->ignore) {
+        OJ_R_FREE(copts->ignore);
+        copts->ignore = NULL;
+    }
+    if (NULL != copts->dump_opts.only) {
+        OJ_R_FREE((void *)copts->dump_opts.only);
+        copts->dump_opts.only = NULL;
+    }
+    if (NULL != copts->dump_opts.except) {
+        OJ_R_FREE((void *)copts->dump_opts.except);
+        copts->dump_opts.except = NULL;
+    }
+    // The str_rx chain is detached from the defaults when the owner is created
+    // so the chain, if there is one, was built for this options struct alone.
+    oj_rxclass_cleanup(&copts->str_rx);
+}
+
+// Mark the Ruby objects held by an owned options struct. A long lived object
+// such as a writer or an encoder can be the only reference to the classes in
+// the options so without this they could be collected while the owner is still
+// alive leaving the options with dangling VALUEs.
+void oj_options_mark(Options copts) {
+    if (Qnil != copts->hash_class) {
+        rb_gc_mark(copts->hash_class);
+    }
+    if (Qnil != copts->array_class) {
+        rb_gc_mark(copts->array_class);
+    }
+    if (NULL != copts->ignore) {
+        VALUE *vp;
+
+        for (vp = copts->ignore; Qnil != *vp; vp++) {
+            rb_gc_mark(*vp);
+        }
+    }
+}
+
 static int match_string_cb(VALUE key, VALUE value, VALUE rx) {
     RxClass rc = (RxClass)rx;
 
@@ -1444,6 +1608,7 @@ static VALUE dump_ensure(VALUE a) {
     volatile struct dump_arg *arg = (void *)a;
 
     oj_out_free(arg->out);
+    oj_free_call_options(arg->copts);
 
     return Qnil;
 }
@@ -1559,6 +1724,7 @@ static VALUE to_file(int argc, VALUE *argv, VALUE self) {
         oj_parse_options(argv[2], &copts);
     }
     oj_write_obj_to_file(argv[1], StringValuePtr(*argv), &copts);
+    oj_free_call_options(&copts);
 
     return Qnil;
 }
@@ -1580,6 +1746,7 @@ static VALUE to_stream(int argc, VALUE *argv, VALUE self) {
         oj_parse_options(argv[2], &copts);
     }
     oj_write_obj_to_stream(argv[1], *argv, &copts);
+    oj_free_call_options(&copts);
 
     return Qnil;
 }

data/ext/oj/oj.h

@@ -256,6 +256,10 @@ extern VALUE oj_custom_parse_cstr(int argc, VALUE *argv, char *json, size_t len)
 
 extern bool oj_hash_has_key(VALUE hash, VALUE key);
 extern void oj_parse_options(VALUE ropts, Options copts);
+extern void oj_free_call_options(Options copts);
+extern void oj_options_take_ownership(Options copts);
+extern void oj_options_release(Options copts);
+extern void oj_options_mark(Options copts);
 
 extern void  oj_dump_obj_to_json(VALUE obj, Options copts, Out out);
 extern void  oj_dump_obj_to_json_using_params(VALUE obj, Options copts, Out out, int argc, VALUE *argv);

data/ext/oj/parse.c

@@ -1306,9 +1306,16 @@ CLEANUP:
         OJ_R_FREE(json);
     }
     stack_cleanup(&pi->stack);
-    if (pi->str_rx.head != oj_default_options.str_rx.head) {
-        oj_rxclass_cleanup(&pi->str_rx);
+    // The parsers match against pi->options.str_rx. A :match_string option
+    // builds a chain of its own for this call, which has to be freed. Without
+    // the option the member still aliases the chain owned by the defaults and
+    // must be left alone.
+    if (pi->options.str_rx.head != oj_default_options.str_rx.head) {
+        oj_rxclass_cleanup(&pi->options.str_rx);
+        pi->options.str_rx.head = NULL;
+        pi->options.str_rx.tail = NULL;
     }
+    oj_free_call_options(&pi->options);
     if (err_has(&pi->err)) {
         rb_set_errinfo(Qnil);
         if (Qnil != pi->err_class) {

data/ext/oj/parse.h

@@ -49,7 +49,6 @@ typedef struct _parseInfo {
     VALUE            handler;
     struct _valStack stack;
     CircArray        circ_array;
-    struct _rxClass  str_rx;
     int              expect_value;
     int              max_depth;  // just for the json gem
     VALUE            proc;

data/ext/oj/parser.c

@@ -5,6 +5,7 @@
 #include <fcntl.h>
 
 #include "oj.h"
+#include "simd.h"
 
 #define DEBUG 0
 
@@ -594,9 +595,64 @@ static void big_change(ojParser p) {
     }
 }
 
-static void parse(ojParser p, const byte *json, bool more) {
+// Scan forward over string content, returning the first byte that is not
+// STR_OK in string_map. This is a pure drop-in for the scalar loop
+//
+//     for (; STR_OK == string_map[*b]; b++) {}
+//
+// and must return the exact same stop position. The stop set (bytes whose
+// string_map class is not 'R'/STR_OK) is:
+//
+//     * control bytes  0x00..0x1F  (class '.', also stops at the NUL terminator)
+//     * the quote      0x22 '"'    (class 'z')
+//     * the backslash  0x5C '\'    (class 'A')
+//     * every high byte 0x80..0xFF (UTF-8 lead/continuation, classes M/P/Q/'.')
+//
+// Note this differs from parse.c's string_scan_neon, which stops only on
+// \0 \\ " -- parser.c hands multi-byte UTF-8 to its state machine, so the
+// scanner must stop on the high bytes too. The predictor below is derived from
+// string_map, not copied from parse.c.
+//
+// The NEON path only loads a full 16-byte vector when [b, b+16) stays within
+// [., end), so it never reads past the string's allocation; the sub-16-byte
+// tail (and the whole scan on non-NEON builds) uses the scalar loop, which
+// stops naturally at the guaranteed NUL terminator.
+static inline const byte *oj_scan_str_simd(const byte *b, const byte *end) {
+#ifdef HAVE_SIMD_NEON
+    if (SIMD_NEON == SIMD_Impl) {
+        const uint8x16_t quote  = vdupq_n_u8('"');
+        const uint8x16_t bslash = vdupq_n_u8('\\');
+        const uint8x16_t space  = vdupq_n_u8(0x20);
+        const uint8x16_t high   = vdupq_n_u8(0x80);
+
+        while (b + sizeof(uint8x16_t) <= end) {
+            const uint8x16_t chunk = vld1q_u8((const uint8_t *)b);
+            // special lane == 0xFF for any byte that stops the scan.
+            const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(chunk, quote), vceqq_u8(chunk, bslash)),
+                                                vorrq_u8(vcltq_u8(chunk, space), vcgeq_u8(chunk, high)));
+            // Reduce to a 64-bit mask with 4 bits per lane (same idiom as
+            // parse.c's string_scan_neon) and locate the first set lane.
+            const uint8x8_t res  = vshrn_n_u16(vreinterpretq_u16_u8(special), 4);
+            uint64_t        mask = vget_lane_u64(vreinterpret_u64_u8(res), 0);
+            if (0 != mask) {
+                mask &= 0x8888888888888888ull;
+                return b + (OJ_CTZ64(mask) >> 2);
+            }
+            b += sizeof(uint8x16_t);
+        }
+    }
+#else
+    (void)end;
+#endif
+    for (; STR_OK == string_map[*b]; b++) {
+    }
+    return b;
+}
+
+static void parse(ojParser p, const byte *json, size_t len, bool more) {
     const byte *start;
-    const byte *b = json;
+    const byte *b   = json;
+    const byte *end = json + len;
     int         i;
 
     p->line = 1;
@@ -629,8 +685,7 @@ static void parse(ojParser p, const byte *json, bool more) {
             b++;
             p->key.tail = p->key.head;
             start       = b;
-            for (; STR_OK == string_map[*b]; b++) {
-            }
+            b           = oj_scan_str_simd(b, end);
             buf_append_string(&p->key, (const char *)start, b - start);
             if ('"' == *b) {
                 p->map = colon_map;
@@ -654,8 +709,7 @@ static void parse(ojParser p, const byte *json, bool more) {
             b++;
             start       = b;
             p->buf.tail = p->buf.head;
-            for (; STR_OK == string_map[*b]; b++) {
-            }
+            b           = oj_scan_str_simd(b, end);
             buf_append_string(&p->buf, (const char *)start, b - start);
             if ('"' == *b) {
                 p->cur = b - json;
@@ -905,8 +959,7 @@ static void parse(ojParser p, const byte *json, bool more) {
             break;
         case STR_OK:
             start = b;
-            for (; STR_OK == string_map[*b]; b++) {
-            }
+            b     = oj_scan_str_simd(b, end);
             if (':' == p->next_map[256]) {
                 buf_append_string(&p->key, (const char *)start, b - start);
             } else {
@@ -1419,7 +1472,7 @@ static VALUE parser_parse(VALUE self, VALUE json) {
 
     parser_reset(p);
     p->start(p);
-    parse(p, ptr, false);
+    parse(p, ptr, (size_t)RSTRING_LEN(json), false);
     validate_document_end(p);
 
     return p->result(p);
@@ -1440,7 +1493,7 @@ static VALUE load(VALUE self) {
     while (true) {
         rb_funcall(p->reader, oj_readpartial_id, 2, INT2NUM(16385), rbuf);
         if (0 < RSTRING_LEN(rbuf)) {
-            parse(p, (byte *)StringValuePtr(rbuf), true);
+            parse(p, (byte *)StringValuePtr(rbuf), (size_t)RSTRING_LEN(rbuf), true);
         }
         if (Qtrue == rb_funcall(p->reader, oj_eofq_id, 0)) {
             if (0 < p->depth) {
@@ -1510,7 +1563,7 @@ static VALUE parser_file(VALUE self, VALUE filename) {
     while (true) {
         if (0 < (rsize = read(fd, buf, size))) {
             buf[rsize] = '\0';
-            parse(p, buf, true);
+            parse(p, buf, rsize, true);
         }
         if (rsize <= 0) {
             if (0 != rsize) {

data/ext/oj/rails.c

@@ -623,6 +623,7 @@ static void encoder_free(void *ptr) {
     if (NULL != ptr) {
         Encoder e = (Encoder)ptr;
 
+        oj_options_release(&e->opts);
         if (NULL != e->ropts.table) {
             OJ_R_FREE(e->ropts.table);
         }
@@ -633,10 +634,19 @@ static void encoder_free(void *ptr) {
 static void encoder_mark(void *ptr) {
     if (NULL != ptr) {
         Encoder e = (Encoder)ptr;
+        int     i;
 
+        oj_options_mark(&e->opts);
         if (Qnil != e->arg) {
             rb_gc_mark(e->arg);
         }
+        // The optimized classes in the encoder table are not reachable from
+        // anywhere else once optimize() is called on the encoder itself.
+        for (i = 0; i < e->ropts.len; i++) {
+            if (Qnil != e->ropts.table[i].clas) {
+                rb_gc_mark(e->ropts.table[i].clas);
+            }
+        }
     }
 }
 
@@ -661,6 +671,11 @@ static VALUE encoder_new(int argc, VALUE *argv, VALUE self) {
     Encoder e = OJ_R_ALLOC(struct _encoder);
 
     e->opts = oj_default_options;
+    // Detach from the match_string regexps owned by the defaults before the
+    // options are parsed so that a :match_string option builds a chain of its
+    // own that is freed with the encoder.
+    e->opts.str_rx.head = NULL;
+    e->opts.str_rx.tail = NULL;
     copy_opts(&ropts, &e->ropts);
 
     if (1 <= argc && Qnil != *argv) {
@@ -669,6 +684,7 @@ static VALUE encoder_new(int argc, VALUE *argv, VALUE self) {
         e->arg = rb_hash_new();
     }
     oj_parse_options(e->arg, &e->opts);
+    oj_options_take_ownership(&e->opts);
 
     return TypedData_Wrap_Struct(encoder_class, &oj_encoder_type, e);
 }

data/ext/oj/reader.c

@@ -164,8 +164,14 @@ static VALUE partial_io_cb(VALUE rbuf) {
     }
     str = StringValuePtr(rstr);
     cnt = RSTRING_LEN(rstr);
-    strcpy(reader->tail, str);
-    reader->read_end = reader->tail + cnt;
+    if (cnt > (size_t)(reader->end - reader->tail)) {
+        // A misbehaving IO returned more than the requested number of bytes.
+        // Copying it in with strcpy() would overflow the reader buffer.
+        rb_raise(rb_eIOError, "read returned more than the requested number of bytes");
+    }
+    memcpy(reader->tail, str, cnt);
+    reader->tail[cnt] = '\0';
+    reader->read_end  = reader->tail + cnt;
 
     return Qtrue;
 }
@@ -184,8 +190,14 @@ static VALUE io_cb(VALUE rbuf) {
     }
     str = StringValuePtr(rstr);
     cnt = RSTRING_LEN(rstr);
-    strcpy(reader->tail, str);
-    reader->read_end = reader->tail + cnt;
+    if (cnt > (size_t)(reader->end - reader->tail)) {
+        // A misbehaving IO returned more than the requested number of bytes.
+        // Copying it in with strcpy() would overflow the reader buffer.
+        rb_raise(rb_eIOError, "read returned more than the requested number of bytes");
+    }
+    memcpy(reader->tail, str, cnt);
+    reader->tail[cnt] = '\0';
+    reader->read_end  = reader->tail + cnt;
 
     return Qtrue;
 }

data/ext/oj/rxclass.c

@@ -142,3 +142,19 @@ void oj_rxclass_copy(RxClass src, RxClass dest) {
         }
     }
 }
+
+// Add the Ruby objects in the chain to an array. The regexps and the classes
+// can be referenced from nowhere else, as with a default match_string option,
+// and are collected unless something holds onto them.
+void oj_rxclass_keep(RxClass rc, VALUE keep) {
+    RxC rxc;
+
+    for (rxc = rc->head; NULL != rxc; rxc = rxc->next) {
+        if (Qnil != rxc->rrx) {
+            rb_ary_push(keep, rxc->rrx);
+        }
+        if (Qnil != rxc->clas) {
+            rb_ary_push(keep, rxc->clas);
+        }
+    }
+}

data/ext/oj/rxclass.h

@@ -22,5 +22,6 @@ extern int   oj_rxclass_append(RxClass rc, const char *expr, VALUE clas);
 extern VALUE oj_rxclass_match(RxClass rc, const char *str, size_t len);
 extern void  oj_rxclass_copy(RxClass src, RxClass dest);
 extern void  oj_rxclass_rappend(RxClass rc, VALUE rx, VALUE clas);
+extern void  oj_rxclass_keep(RxClass rc, VALUE keep);
 
 #endif /* OJ_RXCLASS_H */

data/ext/oj/saj.c

@@ -82,7 +82,14 @@ inline static void next_non_white(ParseInfo pi) {
         case '\f':
         case '\n':
         case '\r': break;
-        case '/': skip_comment(pi); break;
+        case '/':
+            skip_comment(pi);
+            /* A comment that is not terminated by a newline ends on the null
+             * terminator. Stop here so the loop does not step past it. */
+            if ('\0' == *pi->s) {
+                return;
+            }
+            break;
         default: return;
         }
     }
@@ -118,14 +125,15 @@ static void skip_comment(ParseInfo pi) {
             if ('*' == *pi->s && '/' == *(pi->s + 1)) {
                 pi->s++;
                 return;
-            } else if ('\0' == *pi->s) {
-                if (pi->has_error) {
-                    call_error("comment not terminated", pi, __FILE__, __LINE__);
-                } else {
-                    raise_error("comment not terminated", pi->str, pi->s);
-                }
             }
         }
+        /* The loop only ends on the null terminator so the comment was never
+         * closed. */
+        if (pi->has_error) {
+            call_error("comment not terminated", pi, __FILE__, __LINE__);
+        } else {
+            raise_error("comment not terminated", pi->str, pi->s);
+        }
     } else if ('/' == *pi->s) {
         for (; 1; pi->s++) {
             switch (*pi->s) {
@@ -606,6 +614,19 @@ static void saj_parse(VALUE handler, char *json) {
     }
 }
 
+struct _sajArgs {
+    VALUE handler;
+    char *json;
+};
+
+static VALUE protect_saj_parse(VALUE x) {
+    struct _sajArgs *args = (struct _sajArgs *)x;
+
+    saj_parse(args->handler, args->json);
+
+    return Qnil;
+}
+
 /* call-seq: saj_parse(handler, io)
  *
  * Parses an IO stream or file containing an JSON document. Raises an exception
@@ -670,8 +691,17 @@ oj_saj_parse(int argc, VALUE *argv, VALUE self) {
             rb_raise(rb_eArgError, "saj_parse() expected a String or IO Object.");
         }
     }
-    saj_parse(*argv, json);
-    OJ_R_FREE(json);
-
+    {
+        // saj_parse() raises on a malformed document so the json buffer has to
+        // be freed even when the parse does not return normally.
+        struct _sajArgs args = {*argv, json};
+        int             ex   = 0;
+
+        rb_protect(protect_saj_parse, (VALUE)&args, &ex);
+        OJ_R_FREE(json);
+        if (0 != ex) {
+            rb_jump_tag(ex);
+        }
+    }
     return Qnil;
 }

data/ext/oj/sparse.c

@@ -907,6 +907,15 @@ CLEANUP:
         oj_circ_array_free(pi->circ_array);
     }
     stack_cleanup(&pi->stack);
+    // A :match_string option builds a chain of regexps for this call that has to
+    // be freed. Without the option the member aliases the chain owned by the
+    // defaults and must be left alone.
+    if (pi->options.str_rx.head != oj_default_options.str_rx.head) {
+        oj_rxclass_cleanup(&pi->options.str_rx);
+        pi->options.str_rx.head = NULL;
+        pi->options.str_rx.tail = NULL;
+    }
+    oj_free_call_options(&pi->options);
     if (0 != fd) {
 #ifdef _WIN32
         rb_w32_close(fd);

data/ext/oj/stream_writer.c

@@ -16,15 +16,29 @@ static void stream_writer_free(void *ptr) {
         return;
     }
     sw = (StreamWriter)ptr;
+    oj_options_release(&sw->sw.opts);
     OJ_R_FREE(sw->sw.out.buf);
     OJ_R_FREE(sw->sw.types);
     OJ_R_FREE(ptr);
 }
 
+static void stream_writer_mark(void *ptr) {
+    if (NULL != ptr) {
+        StreamWriter sw = (StreamWriter)ptr;
+
+        oj_options_mark(&sw->sw.opts);
+        // The writer can be the only reference to the stream as in
+        // Oj::StreamWriter.new(StringIO.new) so it must be marked.
+        if (Qnil != sw->stream) {
+            rb_gc_mark(sw->stream);
+        }
+    }
+}
+
 static const rb_data_type_t oj_stream_writer_type = {
     "Oj/stream_writer",
     {
-        NULL,
+        stream_writer_mark,
         stream_writer_free,
         NULL,
     },
@@ -116,6 +130,7 @@ static VALUE stream_writer_new(int argc, VALUE *argv, VALUE self) {
         oj_str_writer_init(&sw->sw, 4096);
         sw->flush_limit = 0;
     }
+    oj_options_take_ownership(&sw->sw.opts);
     sw->sw.out.indent = sw->sw.opts.indent;
     sw->stream        = stream;
     sw->type          = type;

data/ext/oj/string_writer.c

@@ -42,12 +42,18 @@ static void maybe_comma(StrWriter sw) {
 
 // Used by stream writer also.
 void oj_str_writer_init(StrWriter sw, int buf_size) {
-    sw->opts       = oj_default_options;
-    sw->depth      = 0;
-    sw->types      = OJ_R_ALLOC_N(char, 256);
-    sw->types_end  = sw->types + 256;
-    *sw->types     = '\0';
-    sw->keyWritten = 0;
+    sw->opts = oj_default_options;
+    // Detach from the match_string regexps owned by the defaults before the
+    // options are parsed. A :match_string option then builds a chain of its own
+    // that is freed with the writer instead of being appended to the chain the
+    // defaults are using.
+    sw->opts.str_rx.head = NULL;
+    sw->opts.str_rx.tail = NULL;
+    sw->depth            = 0;
+    sw->types            = OJ_R_ALLOC_N(char, 256);
+    sw->types_end        = sw->types + 256;
+    *sw->types           = '\0';
+    sw->keyWritten       = 0;
 
     if (0 == buf_size) {
         buf_size = 4096;
@@ -234,15 +240,22 @@ static void string_writer_free(void *ptr) {
     sw = (StrWriter)ptr;
 
     oj_out_free(&sw->out);
+    oj_options_release(&sw->opts);
 
     OJ_R_FREE(sw->types);
     OJ_R_FREE(ptr);
 }
 
+static void string_writer_mark(void *ptr) {
+    if (NULL != ptr) {
+        oj_options_mark(&((StrWriter)ptr)->opts);
+    }
+}
+
 static const rb_data_type_t oj_string_writer_type = {
     "Oj/string_writer",
     {
-        NULL,
+        string_writer_mark,
         string_writer_free,
         NULL,
     },
@@ -279,6 +292,7 @@ static VALUE str_writer_new(int argc, VALUE *argv, VALUE self) {
     if (1 == argc) {
         oj_parse_options(argv[0], &sw->opts);
     }
+    oj_options_take_ownership(&sw->opts);
     sw->out.argc   = argc - 1;
     sw->out.argv   = argv + 1;
     sw->out.indent = sw->opts.indent;

data/lib/oj/version.rb

@@ -1,4 +1,4 @@
 module Oj
   # Current version of the module.
-  VERSION = '3.17.3'
+  VERSION = '3.17.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oj
 version: !ruby/object:Gem::Version
-  version: 3.17.3
+  version: 3.17.4
 platform: ruby
 authors:
 - Peter Ohler