json 2.20.0 → 2.21.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24e3bc40d587f5a4c001b5f4c7da30170bdf06f5401f2aa1928469c8fb3860e2
-  data.tar.gz: b668789b5f3a56d7db311eb4ec9d10b6d236debab23016fabd1682316d50ae50
+  metadata.gz: ea6afdf75f49e474e408c24daae4856891310979fbb10aa34ead66ad2fbf3321
+  data.tar.gz: 19eb129df23cde0ac607fa6ddf909d6ea5eaef0035b2180eea4c2c8a7a5c5ff0
 SHA512:
-  metadata.gz: aaa62a65724e3caa24797ce93ed3ac710a0186a1fd2db990a20fddb10eb99c71d92a967d0bed9d5086cb5b203c5dda0793faa461fda1e5c6963c62da5b4db3dc
-  data.tar.gz: fc44e571dcb662a35a36165d7f7afbb94b11ab8d5660ab1a65905961b7548260152afbb3286fd233223ea5435b6bb62135a3b67b30bd26ca0cd8114e6921c980
+  metadata.gz: c2212cdac41569ebd41e83e6d1ea3324e944f6a8db0070a2805f2213da7341a05cc8eab4dc6d77852f30d31a8951f36d435fdb0aff647776eb8f42cee25113ef
+  data.tar.gz: b16fdb8a1f5384aa6e9064c7884c0f7ec96632bc2c1c6324cc2e902459e9ac94559ec691cf45dea30fc374fab74ce4e3a82f8bb9d0ec22e0ddd5357764076942

data/CHANGES.md

@@ -2,6 +2,22 @@
 
 ### Unreleased
 
+### 2026-07-31 (2.21.2)
+
+* Fix a use-after-free bug in `JSON::ResumableParser`. [GHSA-9hj4-r449-hfvc].
+
+### 2026-07-13 (2.21.1)
+
+* Fix a compilation issue on Window and Microsoft Visual C++.
+
+### 2026-07-12 (2.21.0)
+
+* `JSON.generate` now accept a `sort_keys` option, which takes either a boolean or a block.
+* Added `#empty?` and `#partial_value?` methods on `JSON::ResumableParser`.
+* Numerous correctness and performance fixes for `JSON::ResumableParser`.
+* Avoid triggering Ruby's `float out of range` warning when parsing out of range numbers.
+* Declare C types with Ruby 4.1 `RUBY_TYPED_THREAD_SAFE_FREE`.
+
 ### 2026-06-23 (2.20.0)
 
 * Both C and Java parsers are no longer recursive, so parsing very deep documents with `max_nesting: false` will no longer

data/README.md

@@ -85,7 +85,7 @@ Both of these behavior can be disabled using the `strict: true` option:
 
 ```ruby
 JSON.generate(Object.new, strict: true) # => Object not allowed in JSON (JSON::GeneratorError)
-JSON.generate(Position.new(1, 2)) # => Position not allowed in JSON (JSON::GeneratorError)
+JSON.generate(Position.new(1, 2), strict: true) # => Position not allowed in JSON (JSON::GeneratorError)
 ```
 
 ## JSON::Coder
@@ -117,13 +117,13 @@ It is also called for objects that do have a JSON equivalent, but are used as Ha
 as well as for strings that aren't valid UTF-8:
 
 ```ruby
-coder = JSON::Combining.new do |object, is_object_key|
+coder = JSON::Coder.new do |object, is_object_key|
   case object
   when String
-    if !string.valid_encoding? || string.encoding != Encoding::UTF_8
-      Base64.encode64(string)
+    if !object.valid_encoding? || object.encoding != Encoding::UTF_8
+      Base64.encode64(object)
     else
-      string
+      object
     end
   else
     object

data/ext/json/ext/generator/generator.c

@@ -34,13 +34,14 @@ typedef struct JSON_Generator_StateStruct {
     bool ascii_only;
     bool script_safe;
     bool strict;
+    VALUE sort_keys;
 } JSON_Generator_State;
 
-static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8;
+static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8, default_sort_keys_proc;
 
 static ID i_to_s, i_to_json, i_new, i_encode;
 static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key,
-             sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_escape_slash, sym_strict, sym_as_json;
+             sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_escape_slash, sym_strict, sym_as_json, sym_sort_keys;
 
 
 #define GET_STATE_TO(self, state) \
@@ -709,6 +710,7 @@ static void State_mark(void *ptr)
     rb_gc_mark_movable(state->object_nl);
     rb_gc_mark_movable(state->array_nl);
     rb_gc_mark_movable(state->as_json);
+    rb_gc_mark_movable(state->sort_keys);
 }
 
 static void State_compact(void *ptr)
@@ -720,6 +722,7 @@ static void State_compact(void *ptr)
     state->object_nl = rb_gc_location(state->object_nl);
     state->array_nl = rb_gc_location(state->array_nl);
     state->as_json = rb_gc_location(state->as_json);
+    state->sort_keys = rb_gc_location(state->sort_keys);
 }
 
 static size_t State_memsize(const void *ptr)
@@ -739,7 +742,7 @@ static const rb_data_type_t JSON_Generator_State_type = {
         .dsize = State_memsize,
         .dcompact = State_compact,
     },
-    .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
+    .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
 };
 
 static void state_init(JSON_Generator_State *state)
@@ -769,6 +772,7 @@ static void vstate_spill(struct generate_json_data *data)
     RB_OBJ_WRITTEN(vstate, Qundef, state->object_nl);
     RB_OBJ_WRITTEN(vstate, Qundef, state->array_nl);
     RB_OBJ_WRITTEN(vstate, Qundef, state->as_json);
+    RB_OBJ_WRITTEN(vstate, Qundef, state->sort_keys);
 }
 
 static inline VALUE json_call_to_json(struct generate_json_data *data, VALUE obj)
@@ -1050,6 +1054,11 @@ static inline long increase_depth(struct generate_json_data *data)
 
 static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
+    if (RB_UNLIKELY(data->state->sort_keys)) {
+        obj = rb_proc_call_with_block(data->state->sort_keys, 1, &obj, Qnil);
+        Check_Type(obj, T_HASH);
+    }
+
     long depth = increase_depth(data);
 
     if (RHASH_SIZE(obj) == 0) {
@@ -1376,6 +1385,7 @@ static VALUE cState_init_copy(VALUE obj, VALUE orig)
     RB_OBJ_WRITTEN(obj, Qundef, objState->object_nl);
     RB_OBJ_WRITTEN(obj, Qundef, objState->array_nl);
     RB_OBJ_WRITTEN(obj, Qundef, objState->as_json);
+    RB_OBJ_WRITTEN(obj, Qundef, objState->sort_keys);
 
     return obj;
 }
@@ -1722,6 +1732,55 @@ static VALUE cState_ascii_only_set(VALUE self, VALUE enable)
     return Qnil;
 }
 
+static VALUE cState_set_default_sort_keys_proc(VALUE self, VALUE proc)
+{
+    if (!rb_obj_is_proc(proc)) {
+        rb_raise(rb_eTypeError, "sort_key_proc must be a Proc");
+    }
+    return default_sort_keys_proc = proc;
+}
+
+static VALUE normalize_sort_keys(VALUE value)
+{
+    if (rb_obj_is_proc(value)) {
+        return value;
+    } else if (value == Qtrue) {
+        return default_sort_keys_proc;
+    } else if (RTEST(value)) {
+        rb_raise(rb_eTypeError, "The `sort_keys` argument must be a boolean or a Proc");
+    } else {
+        return Qfalse;
+    }
+}
+
+/*
+ * call-seq: sort_keys
+ *
+ * Get the value of sort_keys.
+ */
+static VALUE cState_sort_keys_p(VALUE self)
+{
+    GET_STATE(self);
+    return state->sort_keys;
+}
+
+/*
+ * call-seq: sort_keys=(value)
+ *
+ * value is a boolean or a proc. If the value is the boolean true, object keys
+ * will be sorted lexicographically in ascending order.
+ *
+ * If the value is a proc, it receives the entire Hash and must return a Hash
+ * with its pairs in the desired order, allowing for arbitrary sorting.
+ */
+static VALUE cState_sort_keys_set(VALUE self, VALUE value)
+{
+    rb_check_frozen(self);
+    GET_STATE(self);
+    RB_OBJ_WRITE(self, &state->sort_keys, normalize_sort_keys(value));
+    return Qnil;
+}
+
 static VALUE cState_allow_duplicate_key_p(VALUE self)
 {
     GET_STATE(self);
@@ -1832,6 +1891,9 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg)
         state->as_json_single_arg = proc && rb_proc_arity(proc) == 1;
         state_write_value(data, &state->as_json, proc);
     }
+    else if (key == sym_sort_keys)             {
+        state_write_value(data, &state->sort_keys, normalize_sort_keys(val));
+    }
     return ST_CONTINUE;
 }
 
@@ -1909,6 +1971,8 @@ void Init_generator(void)
     VALUE mExt = rb_define_module_under(mJSON, "Ext");
     VALUE mGenerator = rb_define_module_under(mExt, "Generator");
 
+    rb_global_variable(&default_sort_keys_proc);
+
     rb_global_variable(&eGeneratorError);
     eGeneratorError = rb_path2class("JSON::GeneratorError");
 
@@ -1918,6 +1982,8 @@ void Init_generator(void)
     cState = rb_define_class_under(mGenerator, "State", rb_cObject);
     rb_define_alloc_func(cState, cState_s_allocate);
     rb_define_singleton_method(cState, "from_state", cState_from_state_s, 1);
+    rb_define_singleton_method(cState, "default_sort_keys_proc=", cState_set_default_sort_keys_proc, 1);
+
     rb_define_method(cState, "initialize", cState_initialize, -1);
     rb_define_alias(cState, "initialize", "initialize"); // avoid method redefinition warnings
     rb_define_private_method(cState, "_configure", cState_configure, 1);
@@ -1957,6 +2023,8 @@ void Init_generator(void)
     rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1);
     rb_define_method(cState, "generate", cState_generate, -1);
     rb_define_method(cState, "_generate_no_fallback", cState_generate_no_fallback, -1);
+    rb_define_method(cState, "sort_keys", cState_sort_keys_p, 0);
+    rb_define_method(cState, "sort_keys=", cState_sort_keys_set, 1);
 
     rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0);
 
@@ -1986,6 +2054,7 @@ void Init_generator(void)
     sym_strict = ID2SYM(rb_intern("strict"));
     sym_as_json = ID2SYM(rb_intern("as_json"));
     sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key"));
+    sym_sort_keys = ID2SYM(rb_intern("sort_keys"));
 
     usascii_encindex = rb_usascii_encindex();
     utf8_encindex = rb_utf8_encindex();

data/ext/json/ext/json.h

@@ -31,6 +31,10 @@
 
 /* shims */
 
+#ifndef RUBY_TYPED_THREAD_SAFE_FREE
+#define RUBY_TYPED_THREAD_SAFE_FREE RUBY_TYPED_FREE_IMMEDIATELY
+#endif
+
 #ifndef UNDEF_P
 #define UNDEF_P(val) (val == Qundef)
 #endif

data/ext/json/ext/parser/extconf.rb

@@ -15,22 +15,6 @@ have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2
 have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby
 have_func("ruby_xfree_sized", "ruby.h") # RUBY_VERSION >= 4.1
 
-def have_builtin_func(name, check_expr, opt = "", &b)
-  checking_for checking_message(name.funcall_style, nil, opt) do
-    if try_compile(<<SRC, opt, &b)
-int foo;
-int main() { #{check_expr}; return 0; }
-SRC
-      $defs.push(format("-DHAVE_BUILTIN_%s", name.tr_cpp))
-      true
-    else
-      false
-    end
-  end
-end
-
-have_builtin_func("__builtin_clzll", "__builtin_clzll(0)")
-
 if have_header("x86intrin.h")
   have_func("_lzcnt_u64", "x86intrin.h")
 end
@@ -46,6 +30,12 @@ end
 
 append_cflags("-std=c99")
 
+# Disable function outlining on clang to prevent some of the repeated instructions
+# in json_eat_whitespace being outlined into function calls.
+# Note: This verifies '-mno-outline' is accepted as a valid compiler flag
+# and will not pass it if unsupported.
+append_cflags("-mno-outline")
+
 if enable_config('parser-use-simd', default=!ENV["JSON_DISABLE_SIMD"])
   load __dir__ + "/../simd/conf.rb"
 end

data/ext/json/ext/parser/parser.c

@@ -5,7 +5,10 @@
 static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8;
 static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string;
 
-static ID i_new, i_try_convert, i_uminus, i_encode, i_at_line, i_at_column;
+static ID i_new, i_try_convert, i_encode, i_at_line, i_at_column;
+#ifndef HAVE_RB_STR_TO_INTERNED_STR
+static ID i_uminus;
+#endif
 
 static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments,
              sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names,
@@ -314,7 +317,7 @@ static const rb_data_type_t JSON_Parser_rvalue_stack_type = {
     },
     // We deliberately don't declare rvalue_stack as RUBY_TYPED_WB_PROTECTED
     // because it churns a lot of values so trigering write barriers every time is very costly.
-    .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE,
+    .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE,
 };
 
 static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref)
@@ -511,7 +514,7 @@ static const rb_data_type_t JSON_Parser_frame_stack_type = {
         .dfree = json_frame_stack_free,
         .dsize = json_frame_stack_memsize,
     },
-    .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE,
+    .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE,
 };
 
 static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref)
@@ -586,6 +589,8 @@ static inline char peek(JSON_ParserState *state)
 
 static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out)
 {
+    JSON_ASSERT(!state->parser);
+    JSON_ASSERT(state->cursor);
     JSON_ASSERT(state->cursor <= state->end);
 
     // Redundant but helpful for hardening
@@ -618,10 +623,14 @@ static const unsigned int MAX_DEPRECATIONS = 5;
 
 static void emit_parse_warning(const char *message, JSON_ParserState *state)
 {
-    long line, column;
-    cursor_position(state, &line, &column);
-
-    VALUE warning = rb_sprintf("%s at line %ld column %ld", message, line, column);
+    VALUE warning;
+    if (state->parser) { // line and columns can't be accurate in resumable
+        warning = rb_utf8_str_new_cstr(message);
+    } else {
+        long line, column;
+        cursor_position(state, &line, &column);
+        warning = rb_sprintf("%s at line %ld column %ld", message, line, column);
+    }
     rb_funcall(mJSON, rb_intern("deprecation_warning"), 1, warning);
 }
 
@@ -765,22 +774,34 @@ static const rb_data_type_t JSON_ParserConfig_type;
 
 const char *COMMENT_DEPRECATION_MESSAGE = "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`";
 NOINLINE(static) void
-json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config)
+json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config, const char *resume_pos)
 {
     if (config->on_comment == JSON_RAISE) {
         raise_syntax_error("unexpected token %s", state);
     }
 
     const char *start = state->cursor;
+    // An incomplete comment suspends a resumable parse by rewinding the cursor
+    // and throwing. Callers that already consumed a token not yet committed to
+    // the frame stack pass resume_pos so the rewind re-reads that token too.
+    // Non-resumable error positions keep pointing at the comment either way.
+    const char *rewind_pos = (state->parser && resume_pos) ? resume_pos : start;
     state->cursor++;
 
     switch (peek(state)) {
         case '/': {
-            state->cursor = memchr(state->cursor, '\n', state->end - state->cursor);
-            if (!state->cursor) {
+            const char *newline = memchr(state->cursor, '\n', state->end - state->cursor);
+            if (!newline) {
+                // state->parser marks resumable mode, where the buffer end is only a
+                // chunk boundary: the terminating newline may still arrive, so leave
+                // the comment unterminated instead of consuming to end as a one-shot
+                // parse would.
+                if (state->parser) {
+                    raise_eos_error_at("unterminated comment, expected end of line", state, rewind_pos);
+                }
                 state->cursor = state->end;
             } else {
-                state->cursor++;
+                state->cursor = newline + 1;
             }
             break;
         }
@@ -790,7 +811,7 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config)
             while (true) {
                 const char *next_match = memchr(state->cursor, '*', state->end - state->cursor);
                 if (!next_match) {
-                    raise_eos_error_at("unterminated comment, expected closing '*/'", state, start);
+                    raise_eos_error_at("unterminated comment, expected closing '*/'", state, rewind_pos);
                 }
 
                 state->cursor = next_match + 1;
@@ -802,7 +823,7 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config)
             break;
         }
         default:
-            raise_parse_error_at("unexpected token %s", state, start, eos(state));
+            raise_parse_error_at("unexpected token %s", state, eos(state) ? rewind_pos : start, eos(state));
             break;
     }
 
@@ -813,7 +834,7 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config)
 }
 
 ALWAYS_INLINE(static) void
-json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments)
+json_eat_whitespace_resume_at(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments, const char *resume_pos)
 {
     while (true) {
         switch (peek(state)) {
@@ -848,7 +869,7 @@ json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool inc
                     return;
                 }
 
-                json_eat_comments(state, config);
+                json_eat_comments(state, config, resume_pos);
                 break;
 
             default:
@@ -857,6 +878,12 @@ json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool inc
     }
 }
 
+ALWAYS_INLINE(static) void
+json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments)
+{
+    json_eat_whitespace_resume_at(state, config, include_comments, NULL);
+}
+
 static inline VALUE build_string(const char *start, const char *end, bool intern, bool symbolize)
 {
     if (symbolize) {
@@ -1130,6 +1157,13 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis
     }
 
     if (RB_UNLIKELY(mantissa_digits > 18 || mantissa_digits + exponent < -307)) {
+        // If the value is so small that it definitely underflows to 0.0, return early
+        // to avoid triggering a "Float out of range" warning from rb_cstr_to_dbl.
+        // When mantissa_digits + exponent < -324, value < 10^(-324) < DBL_TRUE_MIN/2,
+        // so it rounds to 0 in IEEE 754 round-to-nearest.
+        if (RB_UNLIKELY(mantissa_digits + exponent < -324)) {
+            return rb_float_new(negative ? -0.0 : 0.0);
+        }
         return json_decode_large_float(start, end - start);
     }
 
@@ -1429,7 +1463,7 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat
     return (int)(state->cursor - start);
 }
 
-static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start)
+static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start, bool resumable)
 {
     bool integer = true;
     const char first_digit = *state->cursor;
@@ -1486,6 +1520,16 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
         }
     }
 
+    // A number touching the end of the buffer may still grow in a later chunk,
+    // so the caller will rewind and wait. Decoding it now would build a value
+    // -- for a long run of digits, an expensive bignum -- only to discard it,
+    // and repeating that on every resumed chunk is quadratic in the number's
+    // length. The digit scan above already advanced the cursor, which is all
+    // the caller needs to detect the incomplete number.
+    if (RB_UNLIKELY(resumable && eos(state))) {
+        return Qundef;
+    }
+
     if (integer) {
         return json_decode_integer(mantissa, mantissa_digits, negative, start, state->cursor);
     }
@@ -1563,6 +1607,13 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo
     JSON_PHASE_VALUE: {
         json_eat_whitespace(state, config, true);
 
+        // A trailing comma lands us here expecting an element but finding the
+        // closing bracket; hand off to ARRAY_COMMA to close. An empty array
+        // closes inline at '[', so this position is only reached after a ','.
+        if (config->allow_trailing_comma && frame->type == JSON_FRAME_ARRAY && peek(state) == ']') {
+            goto JSON_PHASE_ARRAY_COMMA;
+        }
+
         VALUE value;
         const char *value_start = state->cursor;
 
@@ -1603,7 +1654,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo
             case '-': {
                 state->cursor++;
 
-                value = json_parse_number(state, config, true, value_start);
+                value = json_parse_number(state, config, true, value_start, resumable);
 
                 if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) {
                     state->cursor = value_start;
@@ -1626,7 +1677,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo
             }
 
             case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': {
-                value = json_parse_number(state, config, false, value_start);
+                value = json_parse_number(state, config, false, value_start, resumable);
 
                 // Top level numbers are ambiguous when parsing streams, we can't
                 // know if we parsed all the digits if we hit EOS.
@@ -1658,7 +1709,9 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo
 
             case '[': {
                 state->cursor++;
-                json_eat_whitespace(state, config, true);
+                // The '[' is consumed but its frame is only pushed below, so a
+                // comment suspending here must resume from the bracket.
+                json_eat_whitespace_resume_at(state, config, true, value_start);
 
                 const char next = peek(state);
                 if (next == ']') {
@@ -1687,7 +1740,8 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo
 
             case '{': {
                 state->cursor++;
-                json_eat_whitespace(state, config, true);
+                // Same as '[': the frame is only pushed below.
+                json_eat_whitespace_resume_at(state, config, true, value_start);
 
                 if (peek(state) == '}') {
                     state->cursor++;
@@ -1745,6 +1799,13 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo
 
         json_eat_whitespace(state, config, true);
 
+        // A trailing comma lands us here expecting a key but finding the closing
+        // brace; hand off to OBJECT_COMMA to close. An empty object closes inline
+        // at '{', so this position is only reached after a ','.
+        if (config->allow_trailing_comma && peek(state) == '}') {
+            goto JSON_PHASE_OBJECT_COMMA;
+        }
+
         const char *start = state->cursor;
 
         if (RB_LIKELY(peek(state) == '"')) {
@@ -1807,13 +1868,10 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo
 
         if (RB_LIKELY(next_char == ',')) {
             state->cursor++;
-            if (config->allow_trailing_comma) {
-                json_eat_whitespace(state, config, true);
-                if (peek(state) == ']') {
-                    // Trailing comma: stay in COMMA to close on the next iteration.
-                    goto JSON_PHASE_ARRAY_COMMA;
-                }
-            }
+            // Commit the phase before eating the whitespace that follows: an
+            // incomplete comment there would suspend the parse, and a phase not
+            // yet advanced past the ',' would drop it on resume. A trailing comma
+            // is recognized in JSON_PHASE_VALUE once the ']' is in the buffer.
             frame->phase = JSON_PHASE_VALUE;
             goto JSON_PHASE_VALUE;
         } else if (next_char == ']') {
@@ -1852,15 +1910,10 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo
 
         if (RB_LIKELY(next_char == ',')) {
             state->cursor++;
-            json_eat_whitespace(state, config, true);
-
-            if (config->allow_trailing_comma) {
-                if (peek(state) == '}') {
-                    // Trailing comma: stay in COMMA to close on the next iteration.
-                    goto JSON_PHASE_OBJECT_COMMA;
-                }
-            }
-
+            // Commit the phase before eating the whitespace that follows: an
+            // incomplete comment there would suspend the parse, and a phase not
+            // yet advanced past the ',' would drop it on resume. A trailing comma
+            // is recognized in JSON_PHASE_OBJECT_KEY once the '}' is in the buffer.
             frame->phase = JSON_PHASE_OBJECT_KEY;
             goto JSON_PHASE_OBJECT_KEY;
         } else if (next_char == '}') {
@@ -2180,7 +2233,7 @@ static const rb_data_type_t JSON_ParserConfig_type = {
         .dsize = JSON_ParserConfig_memsize,
         .dcompact = JSON_ParserConfig_compact,
     },
-    .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
+    .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
 };
 
 static VALUE cJSON_parser_s_allocate(VALUE klass)
@@ -2265,7 +2318,7 @@ static const rb_data_type_t JSON_ResumableParser_type = {
     // RUBY_TYPED_WB_PROTECTED is deliberately not declared because
     // this is a superset of JSON_Parser_rvalue_stack_type, so we'd need
     // to trigger a lot of write barriers.
-    .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE,
+    .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE,
 };
 
 static VALUE cResumableParser_allocate(VALUE klass)
@@ -2519,6 +2572,7 @@ static VALUE cResumableParser_parse(VALUE self)
     if (eos(&parser->state)) {
         json_str_clear(parser->buffer);
         parser->buffer = Qfalse;
+        parser->state.start = parser->state.cursor = parser->state.end = 0;
     }
     parser->in_use = false;
 
@@ -2615,11 +2669,16 @@ static VALUE cResumableParser_partial_value_body(VALUE self)
         missing_object_value = 1;
     }
 
-    // Copy the value stack as we need to mutate it.
+    // Copy the value stack as we need to mutate it. The collapse loop folds each
+    // open container by popping its entries and pushing the single result, so a
+    // parent always reclaims its child's slot; head exceeds its live size by at
+    // most one, either for the missing-value placeholder pushed below or for the
+    // result of folding an empty innermost container. That one spare slot keeps
+    // rvalue_stack_push from growing (reallocating) this ALLOCV buffer.
     long capa = parser.value_stack.head;
-    parser.value_stack.capa = (capa + missing_object_value);
-    VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, capa + missing_object_value);
-    MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, parser.value_stack.capa);
+    parser.value_stack.capa = capa + 1;
+    VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, parser.value_stack.capa);
+    MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, capa);
     parser.value_stack.ptr = value_stack_buffer;
 
     JSON_ParserState *state = &parser.state;
@@ -2712,7 +2771,7 @@ static VALUE cResumableParser_rest(VALUE self)
 }
 
 /*
- * call-seq: value? -> true or false
+ * call-seq: eos? -> true or false
  *
  * Returns whether the internal buffer has been entirely consumed.
  */
@@ -2722,6 +2781,41 @@ static VALUE cResumableParser_eos_p(VALUE self)
     return eos(&parser->state) ? Qtrue : Qfalse;
 }
 
+/*
+ * call-seq: partial_value? -> true or false
+ *
+ * Returns whether a document is currently under construction: an unclosed
+ * container, a key awaiting its value, etc.
+ *
+ * It answers the same question as <tt>!partial_value.nil?</tt>, but as a
+ * cheap predicate on the parser's internal state, without materializing the
+ * partially parsed Ruby objects:
+ *   parser << '{"a":1,'
+ *   parser.parse # => false
+ *   parser.partial_value? # => true
+ *
+ * A fully parsed document whose value hasn't been retrieved yet is not under
+ * construction: #value? returns true and #partial_value? returns false.
+ */
+static VALUE cResumableParser_partial_value_p(VALUE self)
+{
+    JSON_ResumableParser *parser = cResumableParser_get(self);
+
+    // Mirror of #value?: values on the stack while the document isn't DONE
+    // belong to a partially built document. A container whose first key or
+    // element hasn't been parsed yet has no frame nor value registered (the
+    // tokenizer rewinds to the container start on EOS), so that state is
+    // observable through the buffer (#eos?/#rest) instead, keeping this
+    // predicate consistent with #partial_value returning nil.
+    if (parser->value_stack.head > 0) {
+        json_frame *frame = json_frame_stack_peek(&parser->frames);
+        if (frame->phase != JSON_PHASE_DONE) {
+            return Qtrue;
+        }
+    }
+    return Qfalse;
+}
+
 /*
  * call-seq: parsed_bytes -> integer
  *
@@ -2778,6 +2872,7 @@ void Init_parser(void)
     rb_define_method(cResumableParser, "value", cResumableParser_value, 0);
     rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0);
     rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0);
+    rb_define_method(cResumableParser, "partial_value?", cResumableParser_partial_value_p, 0);
     rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0);
     rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0);
     rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0);
@@ -2812,7 +2907,9 @@ void Init_parser(void)
 
     i_new = rb_intern("new");
     i_try_convert = rb_intern("try_convert");
+#ifndef HAVE_RB_STR_TO_INTERNED_STR
     i_uminus = rb_intern("-@");
+#endif
     i_encode = rb_intern("encode");
     i_at_line = rb_intern("@line");
     i_at_column = rb_intern("@column");

data/lib/json/common.rb

@@ -155,6 +155,15 @@ module JSON
     # Set the module _generator_ to be used by JSON.
     def generator=(generator) # :nodoc:
       old, $VERBOSE = $VERBOSE, nil
+
+      # The default proc used when the +sort_keys+ generation option is +true+.
+      # It returns a new hash with the entries sorted by their keys.
+      sort_keys_proc = ->(hash) { hash.sort.to_h }
+      if defined?(::Ractor) && Ractor.respond_to?(:shareable_lambda)
+        sort_keys_proc = Ractor.shareable_lambda(&sort_keys_proc)
+      end
+      generator::State.default_sort_keys_proc = sort_keys_proc
+
       @generator = generator
       if generator.const_defined?(:GeneratorMethods)
         generator_methods = generator::GeneratorMethods

data/lib/json/ext/generator/state.rb

@@ -54,6 +54,7 @@ module JSON
             strict: strict?,
             depth: depth,
             buffer_initial_length: buffer_initial_length,
+            sort_keys: sort_keys
           }
 
           allow_duplicate_key = allow_duplicate_key?

data/lib/json/ext.rb

@@ -41,5 +41,31 @@ module JSON
     end
   end
 
+  if defined?(ResumableParser) # Not yet available on JRuby
+    class ResumableParser
+      # Returns whether the parser is entirely done: no unconsumed bytes in
+      # the buffer, no document under construction and no parsed value
+      # awaiting retrieval.
+      #
+      # The main use case is detecting a truncated stream once the input is
+      # exhausted:
+      #
+      #   loop do
+      #     begin
+      #       parser << socket.readpartial(4096)
+      #     rescue EOFError
+      #       break
+      #     end
+      #     while parser.parse
+      #       process(parser.value)
+      #     end
+      #   end
+      #   warn "stream was truncated" unless parser.empty?
+      def empty?
+        eos? && !partial_value? && !value?
+      end
+    end
+  end
+
   JSON_LOADED = true unless defined?(JSON::JSON_LOADED)
 end

data/lib/json/truffle_ruby/generator.rb

@@ -111,6 +111,8 @@ module JSON
       # This class is used to create State instances, that are use to hold data
       # while generating a JSON text from a Ruby data structure.
       class State
+        singleton_class.attr_accessor :default_sort_keys_proc # :nodoc:
+
         def self.generate(obj, opts = nil, io = nil)
           new(opts).generate(obj, io)
         end
@@ -164,6 +166,7 @@ module JSON
           @script_safe           = false
           @strict                = false
           @max_nesting           = 100
+          @sort_keys             = false
           configure(opts) if opts
         end
 
@@ -199,6 +202,33 @@ module JSON
         # supported by the JSON spec will raise a JSON::GeneratorError
         attr_accessor :strict
 
+        # Controls key sorting in the generated JSON. If set to +true+, object
+        # keys are sorted by key lexicographically. If set to a Proc, it
+        # receives the entire Hash and must return a Hash with its pairs in the
+        # desired order.
+        attr_reader :sort_keys
+
+        def sort_keys=(value) # :nodoc:
+          type_error = false
+          @sort_keys = case value
+          when Proc
+            value
+          when true
+            State.default_sort_keys_proc
+          when nil, false
+            false
+          else
+            type_error = true
+            false
+          end
+
+          if type_error
+            raise TypeError, "The `sort_keys` argument must be a boolean or a Proc"
+          end
+
+          @sort_keys
+        end
+
         # :stopdoc:
         attr_reader :buffer_initial_length
 
@@ -285,6 +315,7 @@ module JSON
           @allow_nan             = !!opts[:allow_nan]         if opts.key?(:allow_nan)
           @as_json               = opts[:as_json].to_proc     if opts[:as_json]
           @ascii_only            = opts[:ascii_only]          if opts.key?(:ascii_only)
+          self.sort_keys         = opts[:sort_keys]           if opts.key?(:sort_keys)
           @depth                 = opts[:depth] || 0
           @buffer_initial_length ||= opts[:buffer_initial_length]
 
@@ -349,9 +380,13 @@ module JSON
 
           depth = @depth
           if @indent.empty? and @space.empty? and @space_before.empty? and @object_nl.empty? and @array_nl.empty? and
-              !@ascii_only and !@script_safe and @max_nesting == 0 and (!@strict || Symbol === obj)
+              !@ascii_only and !@script_safe and @max_nesting == 0 and (!@strict || Symbol === obj) and !@sort_keys
             result = generate_json(obj, ''.dup)
           else
+            if @sort_keys
+              obj = @sort_keys.call(obj)
+            end
+
             result = obj.to_json(self)
           end
           JSON::TruffleRuby::Generator.valid_utf8?(result) or raise GeneratorError.new(

data/lib/json/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JSON
-  VERSION = '2.20.0'
+  VERSION = '2.21.2'
 end

data/lib/json.rb

@@ -121,9 +121,11 @@ require 'json/common'
 # ====== Input Options
 #
 # Option +max_nesting+ (\Integer) specifies the maximum nesting depth allowed;
-# defaults to +100+; specify +false+ to disable depth checking.
+# defaults to +100+;
+# You can set it to +false+ to disable depth checking entirely, but that is dangerous
+# when parsing untrusted input.
 #
-# With the default, +false+:
+# With the default, +100+:
 #   source = '[0, [1, [2, [3]]]]'
 #   ruby = JSON.parse(source)
 #   ruby # => [0, [1, [2, [3]]]]
@@ -384,6 +386,15 @@ require 'json/common'
 #   # Raises JSON::NestingError (nesting of 2 is too deep):
 #   JSON.generate(obj, max_nesting: 2)
 #
+# With +false+:
+#   obj = []
+#   obj[0] = obj
+#   # Raises  SystemStackError: stack level too deep
+#   JSON.generate(obj, max_nesting: false)
+#
+# Setting +max_nesting+ to +false+ can lead to a stackoverflow and may leave the program
+# in an unrecoverable state. It is discouraged.
+#
 # ====== Escaping Options
 #
 # Options +script_safe+ (boolean) specifies wether <tt>'\u2028'</tt>, <tt>'\u2029'</tt>
@@ -408,7 +419,6 @@ require 'json/common'
 #   to be inserted after each \JSON object; defaults to the empty \String, <tt>''</tt>.
 # - Option +indent+ (\String) specifies the string (usually spaces) to be
 #   used for indentation; defaults to the empty \String, <tt>''</tt>;
-#   defaults to the empty \String, <tt>''</tt>;
 #   has no effect unless options +array_nl+ or +object_nl+ specify newlines.
 # - Option +space+ (\String) specifies a string (usually a space) to be
 #   inserted after the colon in each \JSON object's pair;
@@ -416,6 +426,11 @@ require 'json/common'
 # - Option +space_before+ (\String) specifies a string (usually a space) to be
 #   inserted before the colon in each \JSON object's pair;
 #   defaults to the empty \String, <tt>''</tt>.
+# - Option +sort_keys+ (boolean or \Proc) controls whether and how the keys of a
+#   hash are sorted when generating the output; defaults to <tt>false</tt>.
+#   When +true+, keys are sorted lexicographically. When a \Proc, it receives
+#   the entire \Hash and must return a \Hash with its pairs in the desired
+#   order, allowing for arbitrary sort orders.
 #
 # In this example, +obj+ is used first to generate the shortest
 # \JSON data (no whitespace), then again with all formatting options

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: json
 version: !ruby/object:Gem::Version
-  version: 2.20.0
+  version: 2.21.2
 platform: ruby
 authors:
 - Florian Frank
@@ -84,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.12
+rubygems_version: 4.0.16
 specification_version: 4
 summary: JSON Implementation for Ruby
 test_files: []