msgpack 1.7.2 → 1.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f5b1af6b3a51f5ccc6bcf67c94c1fc6193b02fe01b123e2cfb06a6df9607116
-  data.tar.gz: cc057f24e1ffa4cdc3e331499eb04de4c2383b0657dcf0baeba08300fd20862e
+  metadata.gz: fa4847fba5fd9dfbed01473471b038a0b06a0ee131395c9724a5bceeab7e9823
+  data.tar.gz: e80a7dfd7c92a1eaa76b3604027094f1dff4d65ba46babcf739019423dd37fad
 SHA512:
-  metadata.gz: 3eb06321a534ca9b16e321cc4a71458532578dafe7967314a662223b1fbf4aa93449c98177fa982aa532ce3732ddda4a6d497704df0e9c874da07f378c73595c
-  data.tar.gz: 8e540755e3db9e21d7dfa4354854e8b0486f5a1bbf82c3994c6095022205f7873153d364df9310d8072c481de38ca2b4c3e088e4221c3451ceb9438312489419
+  metadata.gz: 8f6cbd005ef5d490a2d99ccf45208984a6450ed797f8835884ff758bf71e41e02e397dc174801972386728216f147d09d375763fcc4d9eb853b99520ea40eb19
+  data.tar.gz: e0f6aafd4c147075fe7dcde2f54775cbd84b0d4b8c85b660f2569f4f790dfec979ecbef3da032e8a236558678a54dd8fa1e7ce5346c6cb042936d94037b1ec80

data/ChangeLog

@@ -1,3 +1,11 @@
+2024-11-11 1.7.4
+
+* Fixed a potental memory leak when recursive unpacker raise.
+
+2024-10-03 1.7.3
+
+* Limit initial containers pre-allocation to `SHRT_MAX` (32k) entries.
+
 2023-07-18 1.7.2:
 
 * Fix a potential GC bug when packing data using recursive extensions and buffers containing over 512KkiB of data (See #341).

data/README.md

@@ -8,15 +8,24 @@ and typical short strings only require an extra byte in addition to the strings
 If you ever wished to use JSON for convenience (storing an image with metadata) but could
 not for technical reasons (binary data, size, speed...), MessagePack is a perfect replacement.
 
-    require 'msgpack'
-    msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
-    MessagePack.unpack(msg)   #=> [1,2,3]
+```ruby
+require 'msgpack'
+msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
+MessagePack.unpack(msg)   #=> [1,2,3]
+```
+
+Add msgpack to your Gemfile to install with Bundler:
+
+```ruby
+# Gemfile
+gem 'msgpack'
+```
 
-Use RubyGems to install:
+Or, use RubyGems to install:
 
     gem install msgpack
 
-or build msgpack-ruby and install:
+Or, build msgpack-ruby and install from a checked-out msgpack-ruby repository:
 
     bundle
     rake
@@ -27,11 +36,11 @@ or build msgpack-ruby and install:
 
 * Create REST API returing MessagePack using Rails + [RABL](https://github.com/nesquena/rabl)
 * Store objects efficiently serialized by msgpack on memcached or Redis
-  * In fact Redis supports msgpack in [EVAL-scripts](http://redis.io/commands/eval)
+  * In fact Redis supports msgpack in [EVAL-scripts](https://redis.io/docs/latest/commands/eval/)
 * Upload data in efficient format from mobile devices such as smartphones
   * MessagePack works on iPhone/iPad and Android. See also [Objective-C](https://github.com/msgpack/msgpack-objectivec) and [Java](https://github.com/msgpack/msgpack-java) implementations
 * Design a portable protocol to communicate with embedded devices
-  * Check also [Fluentd](http://fluentd.org/) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
+  * Check also [Fluentd](https://www.fluentd.org) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
 * Exchange objects between software components written in different languages
   * You'll need a flexible but efficient format so that components exchange objects while keeping compatibility
 
@@ -128,9 +137,9 @@ being serialized altogether by throwing an exception:
 
 ```ruby
 class Symbol
-    def to_msgpack_ext
-        raise "Serialization of symbols prohibited"
-    end
+  def to_msgpack_ext
+    raise "Serialization of symbols prohibited"
+  end
 end
 
 MessagePack::DefaultFactory.register_type(0x00, Symbol)
@@ -276,8 +285,8 @@ If this directory has Gemfile.lock (generated with MRI), remove it beforehand.
 
 ## Updating documents
 
-Online documents (http://ruby.msgpack.org) is generated from gh-pages branch.
-Following commands update documents in gh-pages branch:
+Online documentation (https://ruby.msgpack.org) is generated from the gh-pages branch.
+To update documents in gh-pages branch:
 
     bundle exec rake doc
     git checkout gh-pages

data/ext/msgpack/buffer.h

@@ -81,20 +81,6 @@ struct msgpack_buffer_chunk_t {
     bool rmem;
 };
 
-union msgpack_buffer_cast_block_t {
-    char buffer[8];
-    uint8_t u8;
-    uint16_t u16;
-    uint32_t u32;
-    uint64_t u64;
-    int8_t i8;
-    int16_t i16;
-    int32_t i32;
-    int64_t i64;
-    float f;
-    double d;
-};
-
 struct msgpack_buffer_t {
     char* read_buffer;
     char* tail_buffer_end;
@@ -107,8 +93,6 @@ struct msgpack_buffer_t {
     char* rmem_end;
     void** rmem_owner;
 
-    union msgpack_buffer_cast_block_t cast_block;
-
     VALUE io;
     VALUE io_buffer;
     ID io_write_all_method;
@@ -383,14 +367,6 @@ static inline size_t msgpack_buffer_skip_nonblock(msgpack_buffer_t* b, size_t le
     return length;
 }
 
-static inline union msgpack_buffer_cast_block_t* msgpack_buffer_read_cast_block(msgpack_buffer_t* b, size_t n)
-{
-    if(!msgpack_buffer_read_all(b, b->cast_block.buffer, n)) {
-        return NULL;
-    }
-    return &b->cast_block;
-}
-
 size_t msgpack_buffer_read_to_string_nonblock(msgpack_buffer_t* b, VALUE string, size_t length);
 
 static inline size_t msgpack_buffer_read_to_string(msgpack_buffer_t* b, VALUE string, size_t length)

data/ext/msgpack/packer.h

@@ -31,15 +31,15 @@ typedef struct msgpack_packer_t msgpack_packer_t;
 struct msgpack_packer_t {
     msgpack_buffer_t buffer;
 
-    bool compatibility_mode;
-    bool has_bigint_ext_type;
-    bool has_symbol_ext_type;
-
     ID to_msgpack_method;
     VALUE to_msgpack_arg;
 
     VALUE buffer_ref;
 
+    bool compatibility_mode;
+    bool has_bigint_ext_type;
+    bool has_symbol_ext_type;
+
     /* options */
     bool comaptibility_mode;
     msgpack_packer_ext_registry_t ext_registry;

data/ext/msgpack/unpacker.c

@@ -20,13 +20,36 @@
 #include "rmem.h"
 #include "extension_value_class.h"
 #include <assert.h>
+#include <limits.h>
 
 #if !defined(HAVE_RB_PROC_CALL_WITH_BLOCK)
 #define rb_proc_call_with_block(recv, argc, argv, block) rb_funcallv(recv, rb_intern("call"), argc, argv)
 #endif
 
+struct protected_proc_call_args {
+    VALUE proc;
+    int argc;
+    VALUE *argv;
+};
+
+static VALUE protected_proc_call_safe(VALUE _args) {
+    struct protected_proc_call_args *args = (struct protected_proc_call_args *)_args;
+
+    return rb_proc_call_with_block(args->proc, args->argc, args->argv, Qnil);
+}
+
+static VALUE protected_proc_call(VALUE proc, int argc, VALUE *argv, int *raised) {
+    struct protected_proc_call_args args = {
+      .proc = proc,
+      .argc = argc,
+      .argv = argv,
+    };
+    return rb_protect(protected_proc_call_safe, (VALUE)&args, raised);
+}
+
 static int RAW_TYPE_STRING = 256;
 static int RAW_TYPE_BINARY = 257;
+static int16_t INITIAL_BUFFER_CAPACITY_MAX = SHRT_MAX;
 
 static msgpack_rmem_t s_stack_rmem;
 
@@ -37,6 +60,11 @@ static inline VALUE rb_hash_new_capa(long capa)
 }
 #endif
 
+static inline int16_t initial_buffer_size(long size)
+{
+    return (size > INITIAL_BUFFER_CAPACITY_MAX) ? INITIAL_BUFFER_CAPACITY_MAX : size;
+}
+
 void msgpack_unpacker_static_init(void)
 {
     assert(sizeof(msgpack_unpacker_stack_entry_t) * MSGPACK_UNPACKER_STACK_CAPACITY <= MSGPACK_RMEM_PAGE_SIZE);
@@ -51,12 +79,9 @@ void msgpack_unpacker_static_destroy(void)
 
 #define HEAD_BYTE_REQUIRED 0xc1
 
-static inline msgpack_unpacker_stack_t* _msgpack_unpacker_new_stack(void) {
-    msgpack_unpacker_stack_t *stack = ZALLOC(msgpack_unpacker_stack_t);
+static inline void _msgpack_unpacker_stack_init(msgpack_unpacker_stack_t *stack) {
     stack->capacity = MSGPACK_UNPACKER_STACK_CAPACITY;
     stack->data = msgpack_rmem_alloc(&s_stack_rmem);
-    /*memset(uk->stack, 0, MSGPACK_UNPACKER_STACK_CAPACITY);*/
-    return stack;
 }
 
 void _msgpack_unpacker_init(msgpack_unpacker_t* uk)
@@ -68,32 +93,32 @@ void _msgpack_unpacker_init(msgpack_unpacker_t* uk)
     uk->last_object = Qnil;
     uk->reading_raw = Qnil;
 
-    uk->stack = _msgpack_unpacker_new_stack();
+    _msgpack_unpacker_stack_init(&uk->stack);
 }
 
 static inline void _msgpack_unpacker_free_stack(msgpack_unpacker_stack_t* stack) {
     if (!msgpack_rmem_free(&s_stack_rmem, stack->data)) {
         rb_bug("Failed to free an rmem pointer, memory leak?");
     }
-    xfree(stack);
+    stack->data = NULL;
+    stack->depth = 0;
 }
 
 void _msgpack_unpacker_destroy(msgpack_unpacker_t* uk)
 {
-    _msgpack_unpacker_free_stack(uk->stack);
+    _msgpack_unpacker_free_stack(&uk->stack);
     msgpack_buffer_destroy(UNPACKER_BUFFER_(uk));
 }
 
 void msgpack_unpacker_mark_stack(msgpack_unpacker_stack_t* stack)
 {
-    while (stack) {
+    if (stack->data) {
         msgpack_unpacker_stack_entry_t* s = stack->data;
         msgpack_unpacker_stack_entry_t* send = stack->data + stack->depth;
         for(; s < send; s++) {
             rb_gc_mark(s->object);
             rb_gc_mark(s->key);
         }
-        stack = stack->parent;
     }
 }
 
@@ -101,7 +126,7 @@ void msgpack_unpacker_mark(msgpack_unpacker_t* uk)
 {
     rb_gc_mark(uk->last_object);
     rb_gc_mark(uk->reading_raw);
-    msgpack_unpacker_mark_stack(uk->stack);
+    msgpack_unpacker_mark_stack(&uk->stack);
     /* See MessagePack_Buffer_wrap */
     /* msgpack_buffer_mark(UNPACKER_BUFFER_(uk)); */
     rb_gc_mark(uk->buffer_ref);
@@ -114,8 +139,8 @@ void _msgpack_unpacker_reset(msgpack_unpacker_t* uk)
 
     uk->head_byte = HEAD_BYTE_REQUIRED;
 
-    /*memset(uk->stack, 0, sizeof(msgpack_unpacker_t) * uk->stack->depth);*/
-    uk->stack->depth = 0;
+    /*memset(uk->stack, 0, sizeof(msgpack_unpacker_t) * uk->stack.depth);*/
+    uk->stack.depth = 0;
     uk->last_object = Qnil;
     uk->reading_raw = Qnil;
     uk->reading_raw_remaining = 0;
@@ -179,7 +204,12 @@ static inline int object_complete_ext(msgpack_unpacker_t* uk, int ext_type, VALU
     if(proc != Qnil) {
         VALUE obj;
         VALUE arg = (str == Qnil ? rb_str_buf_new(0) : str);
-        obj = rb_proc_call_with_block(proc, 1, &arg, Qnil);
+        int raised;
+        obj = protected_proc_call(proc, 1, &arg, &raised);
+        if (raised) {
+            uk->last_object = rb_errinfo();
+            return PRIMITIVE_RECURSIVE_RAISED;
+        }
         return object_complete(uk, obj);
     }
 
@@ -194,35 +224,35 @@ static inline int object_complete_ext(msgpack_unpacker_t* uk, int ext_type, VALU
 /* stack funcs */
 static inline msgpack_unpacker_stack_entry_t* _msgpack_unpacker_stack_entry_top(msgpack_unpacker_t* uk)
 {
-    return &uk->stack->data[uk->stack->depth-1];
+    return &uk->stack.data[uk->stack.depth-1];
 }
 
 static inline int _msgpack_unpacker_stack_push(msgpack_unpacker_t* uk, enum stack_type_t type, size_t count, VALUE object)
 {
     reset_head_byte(uk);
 
-    if(uk->stack->capacity - uk->stack->depth <= 0) {
+    if(uk->stack.capacity - uk->stack.depth <= 0) {
         return PRIMITIVE_STACK_TOO_DEEP;
     }
 
-    msgpack_unpacker_stack_entry_t* next = &uk->stack->data[uk->stack->depth];
+    msgpack_unpacker_stack_entry_t* next = &uk->stack.data[uk->stack.depth];
     next->count = count;
     next->type = type;
     next->object = object;
     next->key = Qnil;
 
-    uk->stack->depth++;
+    uk->stack.depth++;
     return PRIMITIVE_CONTAINER_START;
 }
 
-static inline VALUE msgpack_unpacker_stack_pop(msgpack_unpacker_t* uk)
+static inline size_t msgpack_unpacker_stack_pop(msgpack_unpacker_t* uk)
 {
-    return --uk->stack->depth;
+    return --uk->stack.depth;
 }
 
 static inline bool msgpack_unpacker_stack_is_empty(msgpack_unpacker_t* uk)
 {
-    return uk->stack->depth == 0;
+    return uk->stack.depth == 0;
 }
 
 #ifdef USE_CASE_RANGE
@@ -241,16 +271,29 @@ static inline bool msgpack_unpacker_stack_is_empty(msgpack_unpacker_t* uk)
 
 #endif
 
+union msgpack_buffer_cast_block_t {
+    char buffer[8];
+    uint8_t u8;
+    uint16_t u16;
+    uint32_t u32;
+    uint64_t u64;
+    int8_t i8;
+    int16_t i16;
+    int32_t i32;
+    int64_t i64;
+    float f;
+    double d;
+};
 
 #define READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, n) \
-    union msgpack_buffer_cast_block_t* cb = msgpack_buffer_read_cast_block(UNPACKER_BUFFER_(uk), n); \
-    if(cb == NULL) { \
+    union msgpack_buffer_cast_block_t cb; \
+    if (!msgpack_buffer_read_all(UNPACKER_BUFFER_(uk), (char *)&cb.buffer, n)) { \
         return PRIMITIVE_EOF; \
     }
 
 static inline bool is_reading_map_key(msgpack_unpacker_t* uk)
 {
-    if(uk->stack->depth > 0) {
+    if(uk->stack.depth > 0) {
         msgpack_unpacker_stack_entry_t* top = _msgpack_unpacker_stack_entry_top(uk);
         if(top->type == STACK_TYPE_MAP_KEY) {
             return true;
@@ -305,14 +348,15 @@ static inline int read_raw_body_begin(msgpack_unpacker_t* uk, int raw_type)
             reset_head_byte(uk);
             uk->reading_raw_remaining = 0;
 
-            msgpack_unpacker_stack_t* child_stack = _msgpack_unpacker_new_stack();
-            child_stack->parent = uk->stack;
-            uk->stack = child_stack;
-
-            obj = rb_proc_call_with_block(proc, 1, &uk->self, Qnil);
+            _msgpack_unpacker_stack_push(uk, STACK_TYPE_RECURSIVE, 1, Qnil);
+            int raised;
+            obj = protected_proc_call(proc, 1, &uk->self, &raised);
+            msgpack_unpacker_stack_pop(uk);
 
-            uk->stack = child_stack->parent;
-            _msgpack_unpacker_free_stack(child_stack);
+            if (raised) {
+                uk->last_object = rb_errinfo();
+                return PRIMITIVE_RECURSIVE_RAISED;
+            }
 
             return object_complete(uk, obj);
         }
@@ -375,14 +419,14 @@ static int read_primitive(msgpack_unpacker_t* uk)
         if(count == 0) {
             return object_complete(uk, rb_ary_new());
         }
-        return _msgpack_unpacker_stack_push(uk, STACK_TYPE_ARRAY, count, rb_ary_new2(count));
+        return _msgpack_unpacker_stack_push(uk, STACK_TYPE_ARRAY, count, rb_ary_new2(initial_buffer_size(count)));
 
     SWITCH_RANGE(b, 0x80, 0x8f)  // FixMap
         int count = b & 0x0f;
         if(count == 0) {
             return object_complete(uk, rb_hash_new());
         }
-        return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
+        return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(initial_buffer_size(count)));
 
     SWITCH_RANGE(b, 0xc0, 0xdf)  // Variable
         switch(b) {
@@ -400,8 +444,8 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xc7: // ext 8
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
-                uint8_t length = cb->u8;
-                int ext_type = (signed char) cb->buffer[1];
+                uint8_t length = cb.u8;
+                int ext_type = (signed char) cb.buffer[1];
                 if(length == 0) {
                     return object_complete_ext(uk, ext_type, Qnil);
                 }
@@ -412,8 +456,8 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xc8: // ext 16
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 3);
-                uint16_t length = _msgpack_be16(cb->u16);
-                int ext_type = (signed char) cb->buffer[2];
+                uint16_t length = _msgpack_be16(cb.u16);
+                int ext_type = (signed char) cb.buffer[2];
                 if(length == 0) {
                     return object_complete_ext(uk, ext_type, Qnil);
                 }
@@ -424,8 +468,8 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xc9: // ext 32
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 5);
-                uint32_t length = _msgpack_be32(cb->u32);
-                int ext_type = (signed char) cb->buffer[4];
+                uint32_t length = _msgpack_be32(cb.u32);
+                int ext_type = (signed char) cb.buffer[4];
                 if(length == 0) {
                     return object_complete_ext(uk, ext_type, Qnil);
                 }
@@ -436,77 +480,77 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xca:  // float
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
-                cb->u32 = _msgpack_be_float(cb->u32);
-                return object_complete(uk, rb_float_new(cb->f));
+                cb.u32 = _msgpack_be_float(cb.u32);
+                return object_complete(uk, rb_float_new(cb.f));
             }
 
         case 0xcb:  // double
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 8);
-                cb->u64 = _msgpack_be_double(cb->u64);
-                return object_complete(uk, rb_float_new(cb->d));
+                cb.u64 = _msgpack_be_double(cb.u64);
+                return object_complete(uk, rb_float_new(cb.d));
             }
 
         case 0xcc:  // unsigned int  8
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
-                uint8_t u8 = cb->u8;
+                uint8_t u8 = cb.u8;
                 return object_complete(uk, INT2NUM((int)u8));
             }
 
         case 0xcd:  // unsigned int 16
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
-                uint16_t u16 = _msgpack_be16(cb->u16);
+                uint16_t u16 = _msgpack_be16(cb.u16);
                 return object_complete(uk, INT2NUM((int)u16));
             }
 
         case 0xce:  // unsigned int 32
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
-                uint32_t u32 = _msgpack_be32(cb->u32);
+                uint32_t u32 = _msgpack_be32(cb.u32);
                 return object_complete(uk, ULONG2NUM(u32)); // long at least 32 bits
             }
 
         case 0xcf:  // unsigned int 64
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 8);
-                uint64_t u64 = _msgpack_be64(cb->u64);
+                uint64_t u64 = _msgpack_be64(cb.u64);
                 return object_complete(uk, rb_ull2inum(u64));
             }
 
         case 0xd0:  // signed int  8
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
-                int8_t i8 = cb->i8;
+                int8_t i8 = cb.i8;
                 return object_complete(uk, INT2NUM((int)i8));
             }
 
         case 0xd1:  // signed int 16
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
-                int16_t i16 = _msgpack_be16(cb->i16);
+                int16_t i16 = _msgpack_be16(cb.i16);
                 return object_complete(uk, INT2NUM((int)i16));
             }
 
         case 0xd2:  // signed int 32
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
-                int32_t i32 = _msgpack_be32(cb->i32);
+                int32_t i32 = _msgpack_be32(cb.i32);
                 return object_complete(uk, LONG2NUM(i32)); // long at least 32 bits
             }
 
         case 0xd3:  // signed int 64
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 8);
-                int64_t i64 = _msgpack_be64(cb->i64);
+                int64_t i64 = _msgpack_be64(cb.i64);
                 return object_complete(uk, rb_ll2inum(i64));
             }
 
         case 0xd4:  // fixext 1
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
-                int ext_type = cb->i8;
+                int ext_type = cb.i8;
                 uk->reading_raw_remaining = 1;
                 return read_raw_body_begin(uk, ext_type);
             }
@@ -514,7 +558,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xd5:  // fixext 2
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
-                int ext_type = cb->i8;
+                int ext_type = cb.i8;
                 uk->reading_raw_remaining = 2;
                 return read_raw_body_begin(uk, ext_type);
             }
@@ -522,7 +566,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xd6:  // fixext 4
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
-                int ext_type = cb->i8;
+                int ext_type = cb.i8;
                 uk->reading_raw_remaining = 4;
                 return read_raw_body_begin(uk, ext_type);
             }
@@ -530,7 +574,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xd7:  // fixext 8
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
-                int ext_type = cb->i8;
+                int ext_type = cb.i8;
                 uk->reading_raw_remaining = 8;
                 return read_raw_body_begin(uk, ext_type);
             }
@@ -538,7 +582,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xd8:  // fixext 16
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
-                int ext_type = cb->i8;
+                int ext_type = cb.i8;
                 uk->reading_raw_remaining = 16;
                 return read_raw_body_begin(uk, ext_type);
             }
@@ -547,7 +591,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xd9:  // raw 8 / str 8
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
-                uint8_t count = cb->u8;
+                uint8_t count = cb.u8;
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -556,7 +600,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xda:  // raw 16 / str 16
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
-                uint16_t count = _msgpack_be16(cb->u16);
+                uint16_t count = _msgpack_be16(cb.u16);
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -565,7 +609,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xdb:  // raw 32 / str 32
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
-                uint32_t count = _msgpack_be32(cb->u32);
+                uint32_t count = _msgpack_be32(cb.u32);
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -574,7 +618,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xc4:  // bin 8
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
-                uint8_t count = cb->u8;
+                uint8_t count = cb.u8;
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -583,7 +627,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xc5:  // bin 16
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
-                uint16_t count = _msgpack_be16(cb->u16);
+                uint16_t count = _msgpack_be16(cb.u16);
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -592,7 +636,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xc6:  // bin 32
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
-                uint32_t count = _msgpack_be32(cb->u32);
+                uint32_t count = _msgpack_be32(cb.u32);
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -601,41 +645,41 @@ static int read_primitive(msgpack_unpacker_t* uk)
         case 0xdc:  // array 16
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
-                uint16_t count = _msgpack_be16(cb->u16);
+                uint16_t count = _msgpack_be16(cb.u16);
                 if(count == 0) {
                     return object_complete(uk, rb_ary_new());
                 }
-                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_ARRAY, count, rb_ary_new2(count));
+                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_ARRAY, count, rb_ary_new2(initial_buffer_size(count)));
             }
 
         case 0xdd:  // array 32
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
-                uint32_t count = _msgpack_be32(cb->u32);
+                uint32_t count = _msgpack_be32(cb.u32);
                 if(count == 0) {
                     return object_complete(uk, rb_ary_new());
                 }
-                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_ARRAY, count, rb_ary_new2(count));
+                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_ARRAY, count, rb_ary_new2(initial_buffer_size(count)));
             }
 
         case 0xde:  // map 16
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
-                uint16_t count = _msgpack_be16(cb->u16);
+                uint16_t count = _msgpack_be16(cb.u16);
                 if(count == 0) {
                     return object_complete(uk, rb_hash_new());
                 }
-                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
+                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(initial_buffer_size(count)));
             }
 
         case 0xdf:  // map 32
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
-                uint32_t count = _msgpack_be32(cb->u32);
+                uint32_t count = _msgpack_be32(cb.u32);
                 if(count == 0) {
                     return object_complete(uk, rb_hash_new());
                 }
-                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
+                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(initial_buffer_size(count)));
             }
 
         default:
@@ -661,12 +705,12 @@ int msgpack_unpacker_read_array_header(msgpack_unpacker_t* uk, uint32_t* result_
     } else if(b == 0xdc) {
         /* array 16 */
         READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
-        *result_size = _msgpack_be16(cb->u16);
+        *result_size = _msgpack_be16(cb.u16);
 
     } else if(b == 0xdd) {
         /* array 32 */
         READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
-        *result_size = _msgpack_be32(cb->u32);
+        *result_size = _msgpack_be32(cb.u32);
 
     } else {
         return PRIMITIVE_UNEXPECTED_TYPE;
@@ -689,12 +733,12 @@ int msgpack_unpacker_read_map_header(msgpack_unpacker_t* uk, uint32_t* result_si
     } else if(b == 0xde) {
         /* map 16 */
         READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
-        *result_size = _msgpack_be16(cb->u16);
+        *result_size = _msgpack_be16(cb.u16);
 
     } else if(b == 0xdf) {
         /* map 32 */
         READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
-        *result_size = _msgpack_be32(cb->u32);
+        *result_size = _msgpack_be32(cb.u32);
 
     } else {
         return PRIMITIVE_UNEXPECTED_TYPE;
@@ -740,6 +784,8 @@ int msgpack_unpacker_read(msgpack_unpacker_t* uk, size_t target_stack_depth)
                 }
                 top->type = STACK_TYPE_MAP_KEY;
                 break;
+            case STACK_TYPE_RECURSIVE:
+                return PRIMITIVE_OBJECT_COMPLETE;
             }
             size_t count = --top->count;
 

data/ext/msgpack/unpacker.h

@@ -31,6 +31,7 @@ enum stack_type_t {
     STACK_TYPE_ARRAY,
     STACK_TYPE_MAP_KEY,
     STACK_TYPE_MAP_VALUE,
+    STACK_TYPE_RECURSIVE,
 };
 
 typedef struct {
@@ -44,31 +45,31 @@ struct msgpack_unpacker_stack_t {
     size_t depth;
     size_t capacity;
     msgpack_unpacker_stack_entry_t *data;
-    msgpack_unpacker_stack_t *parent;
 };
 
 struct msgpack_unpacker_t {
     msgpack_buffer_t buffer;
-    msgpack_unpacker_stack_t *stack;
-    unsigned int head_byte;
+    msgpack_unpacker_stack_t stack;
 
     VALUE self;
     VALUE last_object;
 
     VALUE reading_raw;
     size_t reading_raw_remaining;
-    int reading_raw_type;
 
     VALUE buffer_ref;
 
     msgpack_unpacker_ext_registry_t *ext_registry;
 
+    int reading_raw_type;
+    unsigned int head_byte;
+
     /* options */
+    int symbol_ext_type;
     bool symbolize_keys;
     bool freeze;
     bool allow_unknown_ext;
     bool optimized_symbol_ext_type;
-    int symbol_ext_type;
 };
 
 #define UNPACKER_BUFFER_(uk) (&(uk)->buffer)
@@ -119,6 +120,7 @@ static inline void msgpack_unpacker_set_allow_unknown_ext(msgpack_unpacker_t* uk
 #define PRIMITIVE_STACK_TOO_DEEP -3
 #define PRIMITIVE_UNEXPECTED_TYPE -4
 #define PRIMITIVE_UNEXPECTED_EXT_TYPE -5
+#define PRIMITIVE_RECURSIVE_RAISED -6
 
 int msgpack_unpacker_read(msgpack_unpacker_t* uk, size_t target_stack_depth);
 

data/ext/msgpack/unpacker_class.c

@@ -58,14 +58,17 @@ static void Unpacker_mark(void *ptr)
 
 static size_t Unpacker_memsize(const void *ptr)
 {
+    const msgpack_unpacker_t* uk = ptr;
+
     size_t total_size = sizeof(msgpack_unpacker_t);
 
-    const msgpack_unpacker_t* uk = ptr;
     if (uk->ext_registry) {
         total_size += sizeof(msgpack_unpacker_ext_registry_t) / (uk->ext_registry->borrow_count + 1);
     }
 
-    total_size += (uk->stack->depth + 1) * sizeof(msgpack_unpacker_stack_t);
+    if (uk->stack.data) {
+        total_size += (uk->stack.depth + 1) * sizeof(msgpack_unpacker_stack_t);
+    }
 
     return total_size + msgpack_buffer_memsize(&uk->buffer);
 }
@@ -156,20 +159,28 @@ static VALUE Unpacker_allow_unknown_ext_p(VALUE self)
     return uk->allow_unknown_ext ? Qtrue : Qfalse;
 }
 
-NORETURN(static void raise_unpacker_error(int r))
+NORETURN(static void raise_unpacker_error(msgpack_unpacker_t *uk, int r))
 {
+    uk->stack.depth = 0;
     switch(r) {
     case PRIMITIVE_EOF:
         rb_raise(rb_eEOFError, "end of buffer reached");
+        break;
     case PRIMITIVE_INVALID_BYTE:
         rb_raise(eMalformedFormatError, "invalid byte");
+        break;
     case PRIMITIVE_STACK_TOO_DEEP:
         rb_raise(eStackError, "stack level too deep");
+        break;
     case PRIMITIVE_UNEXPECTED_TYPE:
         rb_raise(eUnexpectedTypeError, "unexpected type");
+        break;
     case PRIMITIVE_UNEXPECTED_EXT_TYPE:
-    // rb_bug("unexpected extension type");
         rb_raise(eUnknownExtTypeError, "unexpected extension type");
+        break;
+    case PRIMITIVE_RECURSIVE_RAISED:
+        rb_exc_raise(msgpack_unpacker_get_last_object(uk));
+        break;
     default:
         rb_raise(eUnpackError, "logically unknown error %d", r);
     }
@@ -190,7 +201,7 @@ static VALUE Unpacker_read(VALUE self)
 
     int r = msgpack_unpacker_read(uk, 0);
     if(r < 0) {
-        raise_unpacker_error(r);
+        raise_unpacker_error(uk, r);
     }
 
     return msgpack_unpacker_get_last_object(uk);
@@ -202,7 +213,7 @@ static VALUE Unpacker_skip(VALUE self)
 
     int r = msgpack_unpacker_skip(uk, 0);
     if(r < 0) {
-        raise_unpacker_error(r);
+        raise_unpacker_error(uk, r);
     }
 
     return Qnil;
@@ -214,7 +225,7 @@ static VALUE Unpacker_skip_nil(VALUE self)
 
     int r = msgpack_unpacker_skip_nil(uk);
     if(r < 0) {
-        raise_unpacker_error(r);
+        raise_unpacker_error(uk, r);
     }
 
     if(r) {
@@ -230,7 +241,7 @@ static VALUE Unpacker_read_array_header(VALUE self)
     uint32_t size;
     int r = msgpack_unpacker_read_array_header(uk, &size);
     if(r < 0) {
-        raise_unpacker_error(r);
+        raise_unpacker_error(uk, r);
     }
 
     return ULONG2NUM(size); // long at least 32 bits
@@ -243,7 +254,7 @@ static VALUE Unpacker_read_map_header(VALUE self)
     uint32_t size;
     int r = msgpack_unpacker_read_map_header(uk, &size);
     if(r < 0) {
-        raise_unpacker_error((int)r);
+        raise_unpacker_error(uk, r);
     }
 
     return ULONG2NUM(size); // long at least 32 bits
@@ -270,7 +281,7 @@ static VALUE Unpacker_each_impl(VALUE self)
             if(r == PRIMITIVE_EOF) {
                 return Qnil;
             }
-            raise_unpacker_error(r);
+            raise_unpacker_error(uk, r);
         }
         VALUE v = msgpack_unpacker_get_last_object(uk);
 #ifdef JRUBY
@@ -369,7 +380,7 @@ static VALUE Unpacker_full_unpack(VALUE self)
 
     int r = msgpack_unpacker_read(uk, 0);
     if(r < 0) {
-        raise_unpacker_error(r);
+        raise_unpacker_error(uk, r);
     }
 
     /* raise if extra bytes follow */

data/lib/msgpack/version.rb

@@ -1,5 +1,5 @@
 module MessagePack
-  VERSION = "1.7.2"
+  VERSION = "1.7.4"
   # Note for maintainers:
   #  Don't miss building/releasing the JRuby version (rake buld:java)
   #  See "How to build -java rubygems" in README for more details.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: msgpack
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: 1.7.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-18 00:00:00.000000000 Z
+date: 2024-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -208,7 +208,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: MessagePack, a binary-based efficient data interchange format.