msgpack 1.5.1 → 1.7.2

data/ext/msgpack/unpacker.c

@@ -19,53 +19,48 @@
 #include "unpacker.h"
 #include "rmem.h"
 #include "extension_value_class.h"
+#include <assert.h>
 
-#if !defined(DISABLE_RMEM) && !defined(DISABLE_UNPACKER_STACK_RMEM) && \
-        MSGPACK_UNPACKER_STACK_CAPACITY * MSGPACK_UNPACKER_STACK_SIZE <= MSGPACK_RMEM_PAGE_SIZE
-#define UNPACKER_STACK_RMEM
+#if !defined(HAVE_RB_PROC_CALL_WITH_BLOCK)
+#define rb_proc_call_with_block(recv, argc, argv, block) rb_funcallv(recv, rb_intern("call"), argc, argv)
 #endif
 
 static int RAW_TYPE_STRING = 256;
 static int RAW_TYPE_BINARY = 257;
 
-static ID s_call;
-
-#ifdef UNPACKER_STACK_RMEM
 static msgpack_rmem_t s_stack_rmem;
-#endif
 
-void msgpack_unpacker_static_init()
+#if !defined(HAVE_RB_HASH_NEW_CAPA)
+static inline VALUE rb_hash_new_capa(long capa)
 {
-#ifdef UNPACKER_STACK_RMEM
-    msgpack_rmem_init(&s_stack_rmem);
+  return rb_hash_new();
+}
 #endif
 
-    s_call = rb_intern("call");
+void msgpack_unpacker_static_init(void)
+{
+    assert(sizeof(msgpack_unpacker_stack_entry_t) * MSGPACK_UNPACKER_STACK_CAPACITY <= MSGPACK_RMEM_PAGE_SIZE);
+
+    msgpack_rmem_init(&s_stack_rmem);
 }
 
-void msgpack_unpacker_static_destroy()
+void msgpack_unpacker_static_destroy(void)
 {
-#ifdef UNPACKER_STACK_RMEM
     msgpack_rmem_destroy(&s_stack_rmem);
-#endif
 }
 
 #define HEAD_BYTE_REQUIRED 0xc1
 
 static inline msgpack_unpacker_stack_t* _msgpack_unpacker_new_stack(void) {
-#ifdef UNPACKER_STACK_RMEM
-    return msgpack_rmem_alloc(&s_stack_rmem);
+    msgpack_unpacker_stack_t *stack = ZALLOC(msgpack_unpacker_stack_t);
+    stack->capacity = MSGPACK_UNPACKER_STACK_CAPACITY;
+    stack->data = msgpack_rmem_alloc(&s_stack_rmem);
     /*memset(uk->stack, 0, MSGPACK_UNPACKER_STACK_CAPACITY);*/
-#else
-    /*uk->stack = calloc(MSGPACK_UNPACKER_STACK_CAPACITY, sizeof(msgpack_unpacker_stack_t));*/
-    return xmalloc(MSGPACK_UNPACKER_STACK_CAPACITY * sizeof(msgpack_unpacker_stack_t));
-#endif
+    return stack;
 }
 
-msgpack_unpacker_t* _msgpack_unpacker_new(void)
+void _msgpack_unpacker_init(msgpack_unpacker_t* uk)
 {
-    msgpack_unpacker_t* uk = ZALLOC_N(msgpack_unpacker_t, 1);
-
     msgpack_buffer_init(UNPACKER_BUFFER_(uk));
 
     uk->head_byte = HEAD_BYTE_REQUIRED;
@@ -74,17 +69,13 @@ msgpack_unpacker_t* _msgpack_unpacker_new(void)
     uk->reading_raw = Qnil;
 
     uk->stack = _msgpack_unpacker_new_stack();
-    uk->stack_capacity = MSGPACK_UNPACKER_STACK_CAPACITY;
-
-    return uk;
 }
 
 static inline void _msgpack_unpacker_free_stack(msgpack_unpacker_stack_t* stack) {
-    #ifdef UNPACKER_STACK_RMEM
-        msgpack_rmem_free(&s_stack_rmem, stack);
-    #else
-        xfree(stack);
-    #endif
+    if (!msgpack_rmem_free(&s_stack_rmem, stack->data)) {
+        rb_bug("Failed to free an rmem pointer, memory leak?");
+    }
+    xfree(stack);
 }
 
 void _msgpack_unpacker_destroy(msgpack_unpacker_t* uk)
@@ -93,21 +84,28 @@ void _msgpack_unpacker_destroy(msgpack_unpacker_t* uk)
     msgpack_buffer_destroy(UNPACKER_BUFFER_(uk));
 }
 
+void msgpack_unpacker_mark_stack(msgpack_unpacker_stack_t* stack)
+{
+    while (stack) {
+        msgpack_unpacker_stack_entry_t* s = stack->data;
+        msgpack_unpacker_stack_entry_t* send = stack->data + stack->depth;
+        for(; s < send; s++) {
+            rb_gc_mark(s->object);
+            rb_gc_mark(s->key);
+        }
+        stack = stack->parent;
+    }
+}
+
 void msgpack_unpacker_mark(msgpack_unpacker_t* uk)
 {
     rb_gc_mark(uk->last_object);
     rb_gc_mark(uk->reading_raw);
-
-    msgpack_unpacker_stack_t* s = uk->stack;
-    msgpack_unpacker_stack_t* send = uk->stack + uk->stack_depth;
-    for(; s < send; s++) {
-        rb_gc_mark(s->object);
-        rb_gc_mark(s->key);
-    }
-
+    msgpack_unpacker_mark_stack(uk->stack);
     /* See MessagePack_Buffer_wrap */
     /* msgpack_buffer_mark(UNPACKER_BUFFER_(uk)); */
     rb_gc_mark(uk->buffer_ref);
+    rb_gc_mark(uk->self);
 }
 
 void _msgpack_unpacker_reset(msgpack_unpacker_t* uk)
@@ -116,9 +114,8 @@ void _msgpack_unpacker_reset(msgpack_unpacker_t* uk)
 
     uk->head_byte = HEAD_BYTE_REQUIRED;
 
-    /*memset(uk->stack, 0, sizeof(msgpack_unpacker_t) * uk->stack_depth);*/
-    uk->stack_depth = 0;
-
+    /*memset(uk->stack, 0, sizeof(msgpack_unpacker_t) * uk->stack->depth);*/
+    uk->stack->depth = 0;
     uk->last_object = Qnil;
     uk->reading_raw = Qnil;
     uk->reading_raw_remaining = 0;
@@ -170,6 +167,9 @@ static inline int object_complete_symbol(msgpack_unpacker_t* uk, VALUE object)
 static inline int object_complete_ext(msgpack_unpacker_t* uk, int ext_type, VALUE str)
 {
     if (uk->optimized_symbol_ext_type && ext_type == uk->symbol_ext_type) {
+        if (RB_UNLIKELY(NIL_P(str))) { // empty extension is returned as Qnil
+            return object_complete_symbol(uk, ID2SYM(rb_intern3("", 0, rb_utf8_encoding())));
+        }
         return object_complete_symbol(uk, rb_str_intern(str));
     }
 
@@ -178,7 +178,8 @@ static inline int object_complete_ext(msgpack_unpacker_t* uk, int ext_type, VALU
 
     if(proc != Qnil) {
         VALUE obj;
-        obj = rb_funcall(proc, s_call, 1, str == Qnil ? rb_str_buf_new(0) : str);
+        VALUE arg = (str == Qnil ? rb_str_buf_new(0) : str);
+        obj = rb_proc_call_with_block(proc, 1, &arg, Qnil);
         return object_complete(uk, obj);
     }
 
@@ -191,37 +192,37 @@ static inline int object_complete_ext(msgpack_unpacker_t* uk, int ext_type, VALU
 }
 
 /* stack funcs */
-static inline msgpack_unpacker_stack_t* _msgpack_unpacker_stack_top(msgpack_unpacker_t* uk)
+static inline msgpack_unpacker_stack_entry_t* _msgpack_unpacker_stack_entry_top(msgpack_unpacker_t* uk)
 {
-    return &uk->stack[uk->stack_depth-1];
+    return &uk->stack->data[uk->stack->depth-1];
 }
 
 static inline int _msgpack_unpacker_stack_push(msgpack_unpacker_t* uk, enum stack_type_t type, size_t count, VALUE object)
 {
     reset_head_byte(uk);
 
-    if(uk->stack_capacity - uk->stack_depth <= 0) {
+    if(uk->stack->capacity - uk->stack->depth <= 0) {
         return PRIMITIVE_STACK_TOO_DEEP;
     }
 
-    msgpack_unpacker_stack_t* next = &uk->stack[uk->stack_depth];
+    msgpack_unpacker_stack_entry_t* next = &uk->stack->data[uk->stack->depth];
     next->count = count;
     next->type = type;
     next->object = object;
     next->key = Qnil;
 
-    uk->stack_depth++;
+    uk->stack->depth++;
     return PRIMITIVE_CONTAINER_START;
 }
 
 static inline VALUE msgpack_unpacker_stack_pop(msgpack_unpacker_t* uk)
 {
-    return --uk->stack_depth;
+    return --uk->stack->depth;
 }
 
 static inline bool msgpack_unpacker_stack_is_empty(msgpack_unpacker_t* uk)
 {
-    return uk->stack_depth == 0;
+    return uk->stack->depth == 0;
 }
 
 #ifdef USE_CASE_RANGE
@@ -249,8 +250,8 @@ static inline bool msgpack_unpacker_stack_is_empty(msgpack_unpacker_t* uk)
 
 static inline bool is_reading_map_key(msgpack_unpacker_t* uk)
 {
-    if(uk->stack_depth > 0) {
-        msgpack_unpacker_stack_t* top = _msgpack_unpacker_stack_top(uk);
+    if(uk->stack->depth > 0) {
+        msgpack_unpacker_stack_entry_t* top = _msgpack_unpacker_stack_entry_top(uk);
         if(top->type == STACK_TYPE_MAP_KEY) {
             return true;
         }
@@ -302,25 +303,17 @@ static inline int read_raw_body_begin(msgpack_unpacker_t* uk, int raw_type)
             VALUE obj;
             uk->last_object = Qnil;
             reset_head_byte(uk);
-            size_t ext_size = uk->reading_raw_remaining;
             uk->reading_raw_remaining = 0;
 
-            msgpack_unpacker_stack_t* stack = uk->stack;
-            size_t stack_depth = uk->stack_depth;
-            size_t stack_capacity = uk->stack_capacity;
-
-            uk->stack = _msgpack_unpacker_new_stack();
-            uk->stack_depth = 0;
-            uk->stack_capacity = MSGPACK_UNPACKER_STACK_CAPACITY;
+            msgpack_unpacker_stack_t* child_stack = _msgpack_unpacker_new_stack();
+            child_stack->parent = uk->stack;
+            uk->stack = child_stack;
 
-            obj = rb_funcall(proc, s_call, 1, uk->buffer.owner);
+            obj = rb_proc_call_with_block(proc, 1, &uk->self, Qnil);
 
-            _msgpack_unpacker_free_stack(uk->stack);
-            uk->stack = stack;
-            uk->stack_depth = stack_depth;
-            uk->stack_capacity = stack_capacity;
+            uk->stack = child_stack->parent;
+            _msgpack_unpacker_free_stack(child_stack);
 
-            msgpack_buffer_skip(UNPACKER_BUFFER_(uk), ext_size);
             return object_complete(uk, obj);
         }
     }
@@ -373,9 +366,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
 
     SWITCH_RANGE(b, 0xa0, 0xbf)  // FixRaw / fixstr
         int count = b & 0x1f;
-        if(count == 0) {
-            return object_complete(uk, rb_utf8_str_new_static("", 0));
-        }
         /* read_raw_body_begin sets uk->reading_raw */
         uk->reading_raw_remaining = count;
         return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -392,7 +382,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         if(count == 0) {
             return object_complete(uk, rb_hash_new());
         }
-        return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new());
+        return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
 
     SWITCH_RANGE(b, 0xc0, 0xdf)  // Variable
         switch(b) {
@@ -475,7 +465,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 uint32_t u32 = _msgpack_be32(cb->u32);
-                return object_complete(uk, ULONG2NUM((unsigned long)u32));
+                return object_complete(uk, ULONG2NUM(u32)); // long at least 32 bits
             }
 
         case 0xcf:  // unsigned int 64
@@ -503,7 +493,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 int32_t i32 = _msgpack_be32(cb->i32);
-                return object_complete(uk, LONG2NUM((long)i32));
+                return object_complete(uk, LONG2NUM(i32)); // long at least 32 bits
             }
 
         case 0xd3:  // signed int 64
@@ -558,9 +548,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
                 uint8_t count = cb->u8;
-                if(count == 0) {
-                    return object_complete(uk, rb_utf8_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -570,9 +557,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
                 uint16_t count = _msgpack_be16(cb->u16);
-                if(count == 0) {
-                    return object_complete(uk, rb_utf8_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -582,9 +566,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 uint32_t count = _msgpack_be32(cb->u32);
-                if(count == 0) {
-                    return object_complete(uk, rb_utf8_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -594,9 +575,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
                 uint8_t count = cb->u8;
-                if(count == 0) {
-                    return object_complete(uk, rb_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -606,9 +584,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
                 uint16_t count = _msgpack_be16(cb->u16);
-                if(count == 0) {
-                    return object_complete(uk, rb_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -618,9 +593,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 uint32_t count = _msgpack_be32(cb->u32);
-                if(count == 0) {
-                    return object_complete(uk, rb_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -653,7 +625,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 if(count == 0) {
                     return object_complete(uk, rb_hash_new());
                 }
-                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new());
+                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
             }
 
         case 0xdf:  // map 32
@@ -663,7 +635,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 if(count == 0) {
                     return object_complete(uk, rb_hash_new());
                 }
-                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new());
+                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
             }
 
         default:
@@ -750,7 +722,7 @@ int msgpack_unpacker_read(msgpack_unpacker_t* uk, size_t target_stack_depth)
 
         container_completed:
         {
-            msgpack_unpacker_stack_t* top = _msgpack_unpacker_stack_top(uk);
+            msgpack_unpacker_stack_entry_t* top = _msgpack_unpacker_stack_entry_top(uk);
             switch(top->type) {
             case STACK_TYPE_ARRAY:
                 rb_ary_push(top->object, uk->last_object);
@@ -800,7 +772,7 @@ int msgpack_unpacker_skip(msgpack_unpacker_t* uk, size_t target_stack_depth)
 
         container_completed:
         {
-            msgpack_unpacker_stack_t* top = _msgpack_unpacker_stack_top(uk);
+            msgpack_unpacker_stack_entry_t* top = _msgpack_unpacker_stack_entry_top(uk);
 
             /* this section optimized out */
             // TODO object_complete still creates objects which should be optimized out

data/ext/msgpack/unpacker.h

@@ -21,12 +21,11 @@
 #include "buffer.h"
 #include "unpacker_ext_registry.h"
 
-#ifndef MSGPACK_UNPACKER_STACK_CAPACITY
 #define MSGPACK_UNPACKER_STACK_CAPACITY 128
-#endif
 
 struct msgpack_unpacker_t;
 typedef struct msgpack_unpacker_t msgpack_unpacker_t;
+typedef struct msgpack_unpacker_stack_t msgpack_unpacker_stack_t;
 
 enum stack_type_t {
     STACK_TYPE_ARRAY,
@@ -39,19 +38,21 @@ typedef struct {
     enum stack_type_t type;
     VALUE object;
     VALUE key;
-} msgpack_unpacker_stack_t;
+} msgpack_unpacker_stack_entry_t;
 
-#define MSGPACK_UNPACKER_STACK_SIZE (8+4+8+8)  /* assumes size_t <= 64bit, enum <= 32bit, VALUE <= 64bit */
+struct msgpack_unpacker_stack_t {
+    size_t depth;
+    size_t capacity;
+    msgpack_unpacker_stack_entry_t *data;
+    msgpack_unpacker_stack_t *parent;
+};
 
 struct msgpack_unpacker_t {
     msgpack_buffer_t buffer;
-
+    msgpack_unpacker_stack_t *stack;
     unsigned int head_byte;
 
-    msgpack_unpacker_stack_t* stack;
-    size_t stack_depth;
-    size_t stack_capacity;
-
+    VALUE self;
     VALUE last_object;
 
     VALUE reading_raw;
@@ -82,11 +83,11 @@ enum msgpack_unpacker_object_type {
     TYPE_MAP,
 };
 
-void msgpack_unpacker_static_init();
+void msgpack_unpacker_static_init(void);
 
-void msgpack_unpacker_static_destroy();
+void msgpack_unpacker_static_destroy(void);
 
-msgpack_unpacker_t* _msgpack_unpacker_new(void);
+void _msgpack_unpacker_init(msgpack_unpacker_t*);
 
 void _msgpack_unpacker_destroy(msgpack_unpacker_t* uk);