msgpack 1.5.4 → 1.8.4

data/ext/msgpack/buffer.c

@@ -19,21 +19,15 @@
 #include "buffer.h"
 #include "rmem.h"
 
-#ifndef HAVE_RB_STR_REPLACE
-static ID s_replace;
-#endif
-
 int msgpack_rb_encindex_utf8;
 int msgpack_rb_encindex_usascii;
 int msgpack_rb_encindex_ascii8bit;
 
 ID s_uminus;
 
-#ifndef DISABLE_RMEM
 static msgpack_rmem_t s_rmem;
-#endif
 
-void msgpack_buffer_static_init()
+void msgpack_buffer_static_init(void)
 {
     s_uminus = rb_intern("-@");
 
@@ -41,20 +35,12 @@ void msgpack_buffer_static_init()
     msgpack_rb_encindex_usascii = rb_usascii_encindex();
     msgpack_rb_encindex_ascii8bit = rb_ascii8bit_encindex();
 
-#ifndef DISABLE_RMEM
     msgpack_rmem_init(&s_rmem);
-#endif
-
-#ifndef HAVE_RB_STR_REPLACE
-    s_replace = rb_intern("replace");
-#endif
 }
 
-void msgpack_buffer_static_destroy()
+void msgpack_buffer_static_destroy(void)
 {
-#ifndef DISABLE_RMEM
     msgpack_rmem_destroy(&s_rmem);
-#endif
 }
 
 void msgpack_buffer_init(msgpack_buffer_t* b)
@@ -72,16 +58,16 @@ void msgpack_buffer_init(msgpack_buffer_t* b)
 static void _msgpack_buffer_chunk_destroy(msgpack_buffer_chunk_t* c)
 {
     if(c->mem != NULL) {
-#ifndef DISABLE_RMEM
-        if(!msgpack_rmem_free(&s_rmem, c->mem)) {
+        if(c->rmem) {
+            if(!msgpack_rmem_free(&s_rmem, c->mem)) {
+                rb_bug("Failed to free an rmem pointer, memory leak?");
+            }
+        } else {
             xfree(c->mem);
         }
         /* no needs to update rmem_owner because chunks will not be
          * free()ed (left in free_list) and thus *rmem_owner is
          * always valid. */
-#else
-        xfree(c->mem);
-#endif
     }
     c->first = NULL;
     c->last = NULL;
@@ -108,8 +94,25 @@ void msgpack_buffer_destroy(msgpack_buffer_t* b)
     }
 }
 
-void msgpack_buffer_mark(msgpack_buffer_t* b)
+size_t msgpack_buffer_memsize(const msgpack_buffer_t* b)
+{
+    size_t memsize = 0;
+    msgpack_buffer_chunk_t* c = b->head;
+
+    while(c) {
+        memsize += sizeof(msgpack_buffer_chunk_t);
+        if(c->mapped_string != NO_MAPPED_STRING) {
+            memsize += (c->last - c->first);
+        }
+        c = c->next;
+    }
+
+    return memsize;
+}
+
+void msgpack_buffer_mark(void *ptr)
 {
+    msgpack_buffer_t* b = ptr;
     /* head is always available */
     msgpack_buffer_chunk_t* c = b->head;
     while(c != &b->tail) {
@@ -120,12 +123,19 @@ void msgpack_buffer_mark(msgpack_buffer_t* b)
 
     rb_gc_mark(b->io);
     rb_gc_mark(b->io_buffer);
-
-    rb_gc_mark(b->owner);
 }
 
 bool _msgpack_buffer_shift_chunk(msgpack_buffer_t* b)
 {
+    if(b->rmem_owner == &b->head->mem) {
+        /* the chunk that owns the rmem page is going away and takes the page
+         * with it. don't carve the remaining space out of a page that is
+         * already back in the pool. */
+        b->rmem_end = NULL;
+        b->rmem_last = NULL;
+        b->rmem_owner = NULL;
+    }
+
     _msgpack_buffer_chunk_destroy(b->head);
 
     if(b->head == &b->tail) {
@@ -133,6 +143,9 @@ bool _msgpack_buffer_shift_chunk(msgpack_buffer_t* b)
          * because head should be always available */
         b->tail_buffer_end = NULL;
         b->read_buffer = NULL;
+        b->rmem_end = NULL;
+        b->rmem_last = NULL;
+        b->rmem_owner = NULL;
         return false;
     }
 
@@ -158,7 +171,6 @@ size_t msgpack_buffer_read_to_string_nonblock(msgpack_buffer_t* b, VALUE string,
 {
     size_t avail = msgpack_buffer_top_readable_size(b);
 
-#ifndef DISABLE_BUFFER_READ_REFERENCE_OPTIMIZE
     /* optimize */
     if(length <= avail && RSTRING_LEN(string) == 0 &&
             b->head->mapped_string != NO_MAPPED_STRING &&
@@ -170,7 +182,6 @@ size_t msgpack_buffer_read_to_string_nonblock(msgpack_buffer_t* b, VALUE string,
         _msgpack_buffer_consumed(b, length);
         return length;
     }
-#endif
 
     size_t const length_orig = length;
 
@@ -252,12 +263,26 @@ bool _msgpack_buffer_read_all2(msgpack_buffer_t* b, char* buffer, size_t length)
 
 static inline msgpack_buffer_chunk_t* _msgpack_buffer_alloc_new_chunk(msgpack_buffer_t* b)
 {
-    msgpack_buffer_chunk_t* reuse = b->free_list;
-    if(reuse == NULL) {
-        return xmalloc(sizeof(msgpack_buffer_chunk_t));
+    msgpack_buffer_chunk_t* chunk = b->free_list;
+    if (chunk) {
+        b->free_list = b->free_list->next;
+    } else {
+        chunk = xmalloc(sizeof(msgpack_buffer_chunk_t));
+    }
+    memset(chunk, 0, sizeof(msgpack_buffer_chunk_t));
+    return chunk;
+}
+
+/* b->tail is copied into nc and then rebuilt, so the rmem page pointer moves
+ * to nc. the owner has to follow it: if it kept pointing at b->tail.mem, the
+ * transfer in _msgpack_buffer_chunk_malloc would degenerate into a
+ * self-assignment and the page would be released by nc while the rebuilt tail
+ * is still reading from it. */
+static inline void _msgpack_buffer_transfer_rmem_owner(msgpack_buffer_t* b, msgpack_buffer_chunk_t* nc)
+{
+    if(b->rmem_owner == &b->tail.mem) {
+        b->rmem_owner = &nc->mem;
     }
-    b->free_list = b->free_list->next;
-    return reuse;
 }
 
 static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
@@ -271,6 +296,7 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
         msgpack_buffer_chunk_t* nc = _msgpack_buffer_alloc_new_chunk(b);
 
         *nc = b->tail;
+        _msgpack_buffer_transfer_rmem_owner(b, nc);
         b->head = nc;
         nc->next = &b->tail;
 
@@ -283,36 +309,61 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
 
         msgpack_buffer_chunk_t* nc = _msgpack_buffer_alloc_new_chunk(b);
 
-#ifndef DISABLE_RMEM
-#ifndef DISABLE_RMEM_REUSE_INTERNAL_FRAGMENT
         if(b->rmem_last == b->tail_buffer_end) {
             /* reuse unused rmem space */
             size_t unused = b->tail_buffer_end - b->tail.last;
             b->rmem_last -= unused;
         }
-#endif
-#endif
 
         /* rebuild tail */
         *nc = b->tail;
+        _msgpack_buffer_transfer_rmem_owner(b, nc);
         before_tail->next = nc;
         nc->next = &b->tail;
     }
 }
 
-void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
+static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE string)
 {
-    size_t length = RSTRING_LEN(string);
+    VALUE mapped_string;
+    if(ENCODING_GET_INLINED(string) == msgpack_rb_encindex_ascii8bit && RB_OBJ_FROZEN_RAW(string)) {
+        mapped_string = string;
+    } else {
+        mapped_string = rb_str_dup(string);
+        ENCODING_SET(mapped_string, msgpack_rb_encindex_ascii8bit);
+    }
+
+    _msgpack_buffer_add_new_chunk(b);
+
+    char* data;
+    size_t length;
+    RSTRING_GETMEM(mapped_string, data, length);
+
+    b->tail.first = (char*) data;
+    b->tail.last = (char*) data + length;
+    b->tail.mapped_string = mapped_string;
+    b->tail.mem = NULL;
+
+    /* msgpack_buffer_writable_size should return 0 for mapped chunk */
+    b->tail_buffer_end = b->tail.last;
+
+    /* consider read_buffer */
+    if(b->head == &b->tail) {
+        b->read_buffer = b->tail.first;
+    }
+}
 
+void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
+{
     if(b->io != Qnil) {
         msgpack_buffer_flush(b);
-        if (ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit) {
+        if (ENCODING_GET_INLINED(string) == msgpack_rb_encindex_ascii8bit) {
             rb_funcall(b->io, b->io_write_all_method, 1, string);
         } else {
-            msgpack_buffer_append(b, RSTRING_PTR(string), length);
+            msgpack_buffer_append(b, RSTRING_PTR(string), RSTRING_LEN(string));
         }
     } else {
-        msgpack_buffer_append(b, RSTRING_PTR(string), length);
+       _msgpack_buffer_append_reference(b, string);
     }
 }
 
@@ -320,11 +371,10 @@ static inline void* _msgpack_buffer_chunk_malloc(
         msgpack_buffer_t* b, msgpack_buffer_chunk_t* c,
         size_t required_size, size_t* allocated_size)
 {
-#ifndef DISABLE_RMEM
     if(required_size <= MSGPACK_RMEM_PAGE_SIZE) {
-#ifndef DISABLE_RMEM_REUSE_INTERNAL_FRAGMENT
+        c->rmem = true;
+
         if((size_t)(b->rmem_end - b->rmem_last) < required_size) {
-#endif
             /* alloc new rmem page */
             *allocated_size = MSGPACK_RMEM_PAGE_SIZE;
             char* buffer = msgpack_rmem_alloc(&s_rmem);
@@ -335,8 +385,6 @@ static inline void* _msgpack_buffer_chunk_malloc(
             b->rmem_last = b->rmem_end = buffer + MSGPACK_RMEM_PAGE_SIZE;
 
             return buffer;
-
-#ifndef DISABLE_RMEM_REUSE_INTERNAL_FRAGMENT
         } else {
             /* reuse unused rmem */
             *allocated_size = (size_t)(b->rmem_end - b->rmem_last);
@@ -350,18 +398,13 @@ static inline void* _msgpack_buffer_chunk_malloc(
 
             return buffer;
         }
-#endif
-    }
-#else
-    if(required_size < 72) {
-        required_size = 72;
     }
-#endif
 
     // TODO alignment?
     *allocated_size = required_size;
     void* mem = xmalloc(required_size);
     c->mem = mem;
+    c->rmem = false;
     return mem;
 }
 
@@ -411,11 +454,7 @@ void _msgpack_buffer_expand(msgpack_buffer_t* b, const char* data, size_t length
     size_t capacity = b->tail.last - b->tail.first;
 
     /* can't realloc mapped chunk or rmem page */
-    if(b->tail.mapped_string != NO_MAPPED_STRING
-#ifndef DISABLE_RMEM
-            || capacity <= MSGPACK_RMEM_PAGE_SIZE
-#endif
-            ) {
+    if(b->tail.mapped_string != NO_MAPPED_STRING || capacity <= MSGPACK_RMEM_PAGE_SIZE) {
         /* allocate new chunk */
         _msgpack_buffer_add_new_chunk(b);
 
@@ -613,9 +652,11 @@ size_t _msgpack_buffer_feed_from_io(msgpack_buffer_t* b)
 
 size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string, size_t length)
 {
+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
+
     if(RSTRING_LEN(string) == 0) {
         /* direct read */
-        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(length), string);
+        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(MIN(b->io_buffer_size, length)), string);
         if(ret == Qnil) {
             return 0;
         }
@@ -627,7 +668,7 @@ size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string,
         b->io_buffer = rb_str_buf_new(0);
     }
 
-    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(length), b->io_buffer);
+    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(MIN(b->io_buffer_size, length)), b->io_buffer);
     if(ret == Qnil) {
         return 0;
     }
@@ -635,6 +676,8 @@ size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string,
 
     rb_str_buf_cat(string, (const void*)RSTRING_PTR(b->io_buffer), rl);
     return rl;
+
+#undef MIN
 }
 
 size_t _msgpack_buffer_skip_from_io(msgpack_buffer_t* b, size_t length)

data/ext/msgpack/buffer.h

@@ -78,20 +78,7 @@ struct msgpack_buffer_chunk_t {
     void* mem;
     msgpack_buffer_chunk_t* next;
     VALUE mapped_string;  /* RBString or NO_MAPPED_STRING */
-};
-
-union msgpack_buffer_cast_block_t {
-    char buffer[8];
-    uint8_t u8;
-    uint16_t u16;
-    uint32_t u32;
-    uint64_t u64;
-    int8_t i8;
-    int16_t i16;
-    int32_t i32;
-    int64_t i64;
-    float f;
-    double d;
+    bool rmem;
 };
 
 struct msgpack_buffer_t {
@@ -102,13 +89,9 @@ struct msgpack_buffer_t {
     msgpack_buffer_chunk_t* head;
     msgpack_buffer_chunk_t* free_list;
 
-#ifndef DISABLE_RMEM
     char* rmem_last;
     char* rmem_end;
     void** rmem_owner;
-#endif
-
-    union msgpack_buffer_cast_block_t cast_block;
 
     VALUE io;
     VALUE io_buffer;
@@ -118,25 +101,25 @@ struct msgpack_buffer_t {
     size_t write_reference_threshold;
     size_t read_reference_threshold;
     size_t io_buffer_size;
-
-    VALUE owner;
 };
 
 /*
  * initialization functions
  */
-void msgpack_buffer_static_init();
+void msgpack_buffer_static_init(void);
 
-void msgpack_buffer_static_destroy();
+void msgpack_buffer_static_destroy(void);
 
 void msgpack_buffer_init(msgpack_buffer_t* b);
 
 void msgpack_buffer_destroy(msgpack_buffer_t* b);
 
-void msgpack_buffer_mark(msgpack_buffer_t* b);
+void msgpack_buffer_mark(void* b);
 
 void msgpack_buffer_clear(msgpack_buffer_t* b);
 
+size_t msgpack_buffer_memsize(const msgpack_buffer_t* b);
+
 static inline void msgpack_buffer_set_write_reference_threshold(msgpack_buffer_t* b, size_t length)
 {
     if(length < MSGPACK_BUFFER_STRING_WRITE_REFERENCE_MINIMUM) {
@@ -254,13 +237,14 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string);
 
 static inline size_t msgpack_buffer_append_string(msgpack_buffer_t* b, VALUE string)
 {
-    size_t length = RSTRING_LEN(string);
+    size_t length;
+    char *ptr;
+    RSTRING_GETMEM(string, ptr, length);
 
     if(length > b->write_reference_threshold) {
         _msgpack_buffer_append_long_string(b, string);
-
     } else {
-        msgpack_buffer_append(b, RSTRING_PTR(string), length);
+        msgpack_buffer_append(b, ptr, length);
     }
 
     return length;
@@ -269,14 +253,9 @@ static inline size_t msgpack_buffer_append_string(msgpack_buffer_t* b, VALUE str
 static inline size_t msgpack_buffer_append_string_reference(msgpack_buffer_t* b, VALUE string)
 {
     size_t length = RSTRING_LEN(string);
-
-    if(length > MSGPACK_BUFFER_STRING_WRITE_REFERENCE_MINIMUM) {
+    if (length > 0) {
         _msgpack_buffer_append_long_string(b, string);
-
-    } else {
-        msgpack_buffer_append(b, RSTRING_PTR(string), length);
     }
-
     return length;
 }
 
@@ -389,14 +368,6 @@ static inline size_t msgpack_buffer_skip_nonblock(msgpack_buffer_t* b, size_t le
     return length;
 }
 
-static inline union msgpack_buffer_cast_block_t* msgpack_buffer_read_cast_block(msgpack_buffer_t* b, size_t n)
-{
-    if(!msgpack_buffer_read_all(b, b->cast_block.buffer, n)) {
-        return NULL;
-    }
-    return &b->cast_block;
-}
-
 size_t msgpack_buffer_read_to_string_nonblock(msgpack_buffer_t* b, VALUE string, size_t length);
 
 static inline size_t msgpack_buffer_read_to_string(msgpack_buffer_t* b, VALUE string, size_t length)
@@ -444,7 +415,6 @@ static inline VALUE _msgpack_buffer_refer_head_mapped_string(msgpack_buffer_t* b
 
 static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_t length, bool will_be_frozen, bool utf8)
 {
-#ifndef DISABLE_BUFFER_READ_REFERENCE_OPTIMIZE
     /* optimize */
     if(!will_be_frozen &&
             b->head->mapped_string != NO_MAPPED_STRING &&
@@ -454,7 +424,6 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
         _msgpack_buffer_consumed(b, length);
         return result;
     }
-#endif
 
     VALUE result;
 
@@ -483,7 +452,6 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
         result = rb_str_new(b->read_buffer, length);
     }
 
-#if STR_UMINUS_DEDUPE
     if (will_be_frozen) {
 #if STR_UMINUS_DEDUPE_FROZEN
         // Starting from MRI 2.8 it is preferable to freeze the string
@@ -495,7 +463,6 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
         // frozen.
         result = rb_funcall(result, s_uminus, 0);
     }
-#endif // STR_UMINUS_DEDUPE
     _msgpack_buffer_consumed(b, length);
     return result;
 
@@ -507,4 +474,131 @@ static inline VALUE msgpack_buffer_read_top_as_symbol(msgpack_buffer_t* b, size_
     return rb_str_intern(msgpack_buffer_read_top_as_string(b, length, true, utf8));
 }
 
+// Hash keys are likely to be repeated, and are frozen.
+// As such we can re-use them if we keep a cache of the ones we've seen so far,
+// and save much more expensive lookups into the global fstring table.
+// This cache implementation is deliberately simple, as we're optimizing for compactness,
+// to be able to fit easily embeded inside msgpack_unpacker_t.
+// As such, binary search into a sorted array gives a good tradeoff between compactness and
+// performance.
+#define MSGPACK_KEY_CACHE_CAPACITY 63
+
+typedef struct msgpack_key_cache_t msgpack_key_cache_t;
+struct msgpack_key_cache_t {
+    int length;
+    VALUE entries[MSGPACK_KEY_CACHE_CAPACITY];
+};
+
+static inline VALUE build_interned_string(const char *str, const long length)
+{
+# ifdef HAVE_RB_ENC_INTERNED_STR
+    return rb_enc_interned_str(str, length, rb_utf8_encoding());
+# else
+    VALUE rstring = rb_utf8_str_new(str, length);
+    return rb_funcall(rb_str_freeze(rstring), s_uminus, 0);
+# endif
+}
+
+static inline VALUE build_symbol(const char *str, const long length)
+{
+    return rb_str_intern(build_interned_string(str, length));
+}
+
+static void rvalue_cache_insert_at(msgpack_key_cache_t *cache, int index, VALUE rstring)
+{
+    MEMMOVE(&cache->entries[index + 1], &cache->entries[index], VALUE, cache->length - index);
+    cache->length++;
+    cache->entries[index] = rstring;
+}
+
+static inline int rstring_cache_cmp(const char *str, const long length, VALUE rstring)
+{
+    long rstring_length = RSTRING_LEN(rstring);
+    if (length == rstring_length) {
+        return memcmp(str, RSTRING_PTR(rstring), length);
+    } else {
+        return (int)(length - rstring_length);
+    }
+}
+
+static VALUE rstring_cache_fetch(msgpack_key_cache_t *cache, const char *str, const long length)
+{
+    int low = 0;
+    int high = cache->length - 1;
+    int mid = 0;
+    int last_cmp = 0;
+
+    while (low <= high) {
+        mid = (high + low) >> 1;
+        VALUE entry = cache->entries[mid];
+        last_cmp = rstring_cache_cmp(str, length, entry);
+
+        if (last_cmp == 0) {
+            return entry;
+        } else if (last_cmp > 0) {
+            low = mid + 1;
+        } else {
+            high = mid - 1;
+        }
+    }
+
+    VALUE rstring = build_interned_string(str, length);
+
+    if (cache->length < MSGPACK_KEY_CACHE_CAPACITY) {
+        if (last_cmp > 0) {
+            mid += 1;
+        }
+
+        rvalue_cache_insert_at(cache, mid, rstring);
+    }
+    return rstring;
+}
+
+static VALUE rsymbol_cache_fetch(msgpack_key_cache_t *cache, const char *str, const long length)
+{
+    int low = 0;
+    int high = cache->length - 1;
+    int mid = 0;
+    int last_cmp = 0;
+
+    while (low <= high) {
+        mid = (high + low) >> 1;
+        VALUE entry = cache->entries[mid];
+        last_cmp = rstring_cache_cmp(str, length, rb_sym2str(entry));
+
+        if (last_cmp == 0) {
+            return entry;
+        } else if (last_cmp > 0) {
+            low = mid + 1;
+        } else {
+            high = mid - 1;
+        }
+    }
+
+    VALUE rsymbol = build_symbol(str, length);
+
+    if (cache->length < MSGPACK_KEY_CACHE_CAPACITY) {
+        if (last_cmp > 0) {
+            mid += 1;
+        }
+
+        rvalue_cache_insert_at(cache, mid, rsymbol);
+    }
+    return rsymbol;
+}
+
+static inline VALUE msgpack_buffer_read_top_as_interned_symbol(msgpack_buffer_t* b, msgpack_key_cache_t *cache, size_t length)
+{
+    VALUE result = rsymbol_cache_fetch(cache, b->read_buffer, length);
+    _msgpack_buffer_consumed(b, length);
+    return result;
+}
+
+static inline VALUE msgpack_buffer_read_top_as_interned_string(msgpack_buffer_t* b, msgpack_key_cache_t *cache, size_t length)
+{
+    VALUE result = rstring_cache_fetch(cache, b->read_buffer, length);
+    _msgpack_buffer_consumed(b, length);
+    return result;
+}
+
 #endif