msgpack 1.3.3 → 1.7.2

data/ext/msgpack/buffer.c

@@ -19,42 +19,28 @@
 #include "buffer.h"
 #include "rmem.h"
 
-#ifndef HAVE_RB_STR_REPLACE
-static ID s_replace;
-#endif
-
-#ifdef COMPAT_HAVE_ENCODING  /* see compat.h*/
 int msgpack_rb_encindex_utf8;
 int msgpack_rb_encindex_usascii;
 int msgpack_rb_encindex_ascii8bit;
-#endif
 
-#ifndef DISABLE_RMEM
+ID s_uminus;
+
 static msgpack_rmem_t s_rmem;
-#endif
 
-void msgpack_buffer_static_init()
+void msgpack_buffer_static_init(void)
 {
-#ifdef COMPAT_HAVE_ENCODING
+    s_uminus = rb_intern("-@");
+
     msgpack_rb_encindex_utf8 = rb_utf8_encindex();
     msgpack_rb_encindex_usascii = rb_usascii_encindex();
     msgpack_rb_encindex_ascii8bit = rb_ascii8bit_encindex();
-#endif
 
-#ifndef DISABLE_RMEM
     msgpack_rmem_init(&s_rmem);
-#endif
-
-#ifndef HAVE_RB_STR_REPLACE
-    s_replace = rb_intern("replace");
-#endif
 }
 
-void msgpack_buffer_static_destroy()
+void msgpack_buffer_static_destroy(void)
 {
-#ifndef DISABLE_RMEM
     msgpack_rmem_destroy(&s_rmem);
-#endif
 }
 
 void msgpack_buffer_init(msgpack_buffer_t* b)
@@ -72,16 +58,16 @@ void msgpack_buffer_init(msgpack_buffer_t* b)
 static void _msgpack_buffer_chunk_destroy(msgpack_buffer_chunk_t* c)
 {
     if(c->mem != NULL) {
-#ifndef DISABLE_RMEM
-        if(!msgpack_rmem_free(&s_rmem, c->mem)) {
+        if(c->rmem) {
+            if(!msgpack_rmem_free(&s_rmem, c->mem)) {
+                rb_bug("Failed to free an rmem pointer, memory leak?");
+            }
+        } else {
             xfree(c->mem);
         }
         /* no needs to update rmem_owner because chunks will not be
          * free()ed (left in free_list) and thus *rmem_owner is
          * always valid. */
-#else
-        xfree(c->mem);
-#endif
     }
     c->first = NULL;
     c->last = NULL;
@@ -108,8 +94,25 @@ void msgpack_buffer_destroy(msgpack_buffer_t* b)
     }
 }
 
-void msgpack_buffer_mark(msgpack_buffer_t* b)
+size_t msgpack_buffer_memsize(const msgpack_buffer_t* b)
 {
+    size_t memsize = 0;
+    msgpack_buffer_chunk_t* c = b->head;
+
+    while(c) {
+        memsize += sizeof(msgpack_buffer_chunk_t);
+        if(c->mapped_string != NO_MAPPED_STRING) {
+            memsize += (c->last - c->first);
+        }
+        c = c->next;
+    }
+
+    return memsize;
+}
+
+void msgpack_buffer_mark(void *ptr)
+{
+    msgpack_buffer_t* b = ptr;
     /* head is always available */
     msgpack_buffer_chunk_t* c = b->head;
     while(c != &b->tail) {
@@ -120,8 +123,6 @@ void msgpack_buffer_mark(msgpack_buffer_t* b)
 
     rb_gc_mark(b->io);
     rb_gc_mark(b->io_buffer);
-
-    rb_gc_mark(b->owner);
 }
 
 bool _msgpack_buffer_shift_chunk(msgpack_buffer_t* b)
@@ -158,24 +159,17 @@ size_t msgpack_buffer_read_to_string_nonblock(msgpack_buffer_t* b, VALUE string,
 {
     size_t avail = msgpack_buffer_top_readable_size(b);
 
-#ifndef DISABLE_BUFFER_READ_REFERENCE_OPTIMIZE
     /* optimize */
     if(length <= avail && RSTRING_LEN(string) == 0 &&
             b->head->mapped_string != NO_MAPPED_STRING &&
             length >= b->read_reference_threshold) {
         VALUE s = _msgpack_buffer_refer_head_mapped_string(b, length);
-#ifndef HAVE_RB_STR_REPLACE
-        /* TODO MRI 1.8 */
-        rb_funcall(string, s_replace, 1, s);
-#else
         rb_str_replace(string, s);
-#endif
         /* here doesn't have to call ENCODING_SET because
          * encoding of s is always ASCII-8BIT */
         _msgpack_buffer_consumed(b, length);
         return length;
     }
-#endif
 
     size_t const length_orig = length;
 
@@ -257,12 +251,14 @@ bool _msgpack_buffer_read_all2(msgpack_buffer_t* b, char* buffer, size_t length)
 
 static inline msgpack_buffer_chunk_t* _msgpack_buffer_alloc_new_chunk(msgpack_buffer_t* b)
 {
-    msgpack_buffer_chunk_t* reuse = b->free_list;
-    if(reuse == NULL) {
-        return xmalloc(sizeof(msgpack_buffer_chunk_t));
+    msgpack_buffer_chunk_t* chunk = b->free_list;
+    if (chunk) {
+        b->free_list = b->free_list->next;
+    } else {
+        chunk = xmalloc(sizeof(msgpack_buffer_chunk_t));
     }
-    b->free_list = b->free_list->next;
-    return reuse;
+    memset(chunk, 0, sizeof(msgpack_buffer_chunk_t));
+    return chunk;
 }
 
 static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
@@ -288,15 +284,11 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
 
         msgpack_buffer_chunk_t* nc = _msgpack_buffer_alloc_new_chunk(b);
 
-#ifndef DISABLE_RMEM
-#ifndef DISABLE_RMEM_REUSE_INTERNAL_FRAGMENT
         if(b->rmem_last == b->tail_buffer_end) {
             /* reuse unused rmem space */
             size_t unused = b->tail_buffer_end - b->tail.last;
             b->rmem_last -= unused;
         }
-#endif
-#endif
 
         /* rebuild tail */
         *nc = b->tail;
@@ -307,10 +299,13 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
 
 static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE string)
 {
-    VALUE mapped_string = rb_str_dup(string);
-#ifdef COMPAT_HAVE_ENCODING
-    ENCODING_SET(mapped_string, msgpack_rb_encindex_ascii8bit);
-#endif
+    VALUE mapped_string;
+    if(ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit && RTEST(rb_obj_frozen_p(string))) {
+        mapped_string = string;
+    } else {
+        mapped_string = rb_str_dup(string);
+        ENCODING_SET(mapped_string, msgpack_rb_encindex_ascii8bit);
+    }
 
     _msgpack_buffer_add_new_chunk(b);
 
@@ -333,29 +328,15 @@ static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE s
 
 void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
 {
-    size_t length = RSTRING_LEN(string);
-
     if(b->io != Qnil) {
         msgpack_buffer_flush(b);
-#ifdef COMPAT_HAVE_ENCODING
         if (ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit) {
             rb_funcall(b->io, b->io_write_all_method, 1, string);
-        } else if(!STR_DUP_LIKELY_DOES_COPY(string)) {
-            VALUE s = rb_str_dup(string);
-            ENCODING_SET(s, msgpack_rb_encindex_ascii8bit);
-            rb_funcall(b->io, b->io_write_all_method, 1, s);
         } else {
-            msgpack_buffer_append(b, RSTRING_PTR(string), length);
+            msgpack_buffer_append(b, RSTRING_PTR(string), RSTRING_LEN(string));
         }
-#else
-        rb_funcall(b->io, b->io_write_all_method, 1, string);
-#endif
-
-    } else if(!STR_DUP_LIKELY_DOES_COPY(string)) {
-        _msgpack_buffer_append_reference(b, string);
-
     } else {
-        msgpack_buffer_append(b, RSTRING_PTR(string), length);
+       _msgpack_buffer_append_reference(b, string);
     }
 }
 
@@ -363,11 +344,10 @@ static inline void* _msgpack_buffer_chunk_malloc(
         msgpack_buffer_t* b, msgpack_buffer_chunk_t* c,
         size_t required_size, size_t* allocated_size)
 {
-#ifndef DISABLE_RMEM
     if(required_size <= MSGPACK_RMEM_PAGE_SIZE) {
-#ifndef DISABLE_RMEM_REUSE_INTERNAL_FRAGMENT
+        c->rmem = true;
+
         if((size_t)(b->rmem_end - b->rmem_last) < required_size) {
-#endif
             /* alloc new rmem page */
             *allocated_size = MSGPACK_RMEM_PAGE_SIZE;
             char* buffer = msgpack_rmem_alloc(&s_rmem);
@@ -378,8 +358,6 @@ static inline void* _msgpack_buffer_chunk_malloc(
             b->rmem_last = b->rmem_end = buffer + MSGPACK_RMEM_PAGE_SIZE;
 
             return buffer;
-
-#ifndef DISABLE_RMEM_REUSE_INTERNAL_FRAGMENT
         } else {
             /* reuse unused rmem */
             *allocated_size = (size_t)(b->rmem_end - b->rmem_last);
@@ -393,18 +371,13 @@ static inline void* _msgpack_buffer_chunk_malloc(
 
             return buffer;
         }
-#endif
     }
-#else
-    if(required_size < 72) {
-        required_size = 72;
-    }
-#endif
 
     // TODO alignment?
     *allocated_size = required_size;
     void* mem = xmalloc(required_size);
     c->mem = mem;
+    c->rmem = false;
     return mem;
 }
 
@@ -454,11 +427,7 @@ void _msgpack_buffer_expand(msgpack_buffer_t* b, const char* data, size_t length
     size_t capacity = b->tail.last - b->tail.first;
 
     /* can't realloc mapped chunk or rmem page */
-    if(b->tail.mapped_string != NO_MAPPED_STRING
-#ifndef DISABLE_RMEM
-            || capacity <= MSGPACK_RMEM_PAGE_SIZE
-#endif
-            ) {
+    if(b->tail.mapped_string != NO_MAPPED_STRING || capacity <= MSGPACK_RMEM_PAGE_SIZE) {
         /* allocate new chunk */
         _msgpack_buffer_add_new_chunk(b);
 
@@ -631,13 +600,13 @@ size_t msgpack_buffer_flush_to_io(msgpack_buffer_t* b, VALUE io, ID write_method
 size_t _msgpack_buffer_feed_from_io(msgpack_buffer_t* b)
 {
     if(b->io_buffer == Qnil) {
-        b->io_buffer = rb_funcall(b->io, b->io_partial_read_method, 1, LONG2NUM(b->io_buffer_size));
+        b->io_buffer = rb_funcall(b->io, b->io_partial_read_method, 1, SIZET2NUM(b->io_buffer_size));
         if(b->io_buffer == Qnil) {
             rb_raise(rb_eEOFError, "IO reached end of file");
         }
         StringValue(b->io_buffer);
     } else {
-        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(b->io_buffer_size), b->io_buffer);
+        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(b->io_buffer_size), b->io_buffer);
         if(ret == Qnil) {
             rb_raise(rb_eEOFError, "IO reached end of file");
         }
@@ -656,9 +625,11 @@ size_t _msgpack_buffer_feed_from_io(msgpack_buffer_t* b)
 
 size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string, size_t length)
 {
+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
+
     if(RSTRING_LEN(string) == 0) {
         /* direct read */
-        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(length), string);
+        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(MIN(b->io_buffer_size, length)), string);
         if(ret == Qnil) {
             return 0;
         }
@@ -670,7 +641,7 @@ size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string,
         b->io_buffer = rb_str_buf_new(0);
     }
 
-    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(length), b->io_buffer);
+    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(MIN(b->io_buffer_size, length)), b->io_buffer);
     if(ret == Qnil) {
         return 0;
     }
@@ -678,6 +649,8 @@ size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string,
 
     rb_str_buf_cat(string, (const void*)RSTRING_PTR(b->io_buffer), rl);
     return rl;
+
+#undef MIN
 }
 
 size_t _msgpack_buffer_skip_from_io(msgpack_buffer_t* b, size_t length)
@@ -686,7 +659,7 @@ size_t _msgpack_buffer_skip_from_io(msgpack_buffer_t* b, size_t length)
         b->io_buffer = rb_str_buf_new(0);
     }
 
-    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(length), b->io_buffer);
+    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(length), b->io_buffer);
     if(ret == Qnil) {
         return 0;
     }

data/ext/msgpack/buffer.h

@@ -49,11 +49,15 @@
 
 #define NO_MAPPED_STRING ((VALUE)0)
 
-#ifdef COMPAT_HAVE_ENCODING  /* see compat.h*/
+#ifndef RB_ENC_INTERNED_STR_NULL_CHECK
+#define RB_ENC_INTERNED_STR_NULL_CHECK 0
+#endif
+
 extern int msgpack_rb_encindex_utf8;
 extern int msgpack_rb_encindex_usascii;
 extern int msgpack_rb_encindex_ascii8bit;
-#endif
+
+extern ID s_uminus;
 
 struct msgpack_buffer_chunk_t;
 typedef struct msgpack_buffer_chunk_t msgpack_buffer_chunk_t;
@@ -74,6 +78,7 @@ struct msgpack_buffer_chunk_t {
     void* mem;
     msgpack_buffer_chunk_t* next;
     VALUE mapped_string;  /* RBString or NO_MAPPED_STRING */
+    bool rmem;
 };
 
 union msgpack_buffer_cast_block_t {
@@ -98,11 +103,9 @@ struct msgpack_buffer_t {
     msgpack_buffer_chunk_t* head;
     msgpack_buffer_chunk_t* free_list;
 
-#ifndef DISABLE_RMEM
     char* rmem_last;
     char* rmem_end;
     void** rmem_owner;
-#endif
 
     union msgpack_buffer_cast_block_t cast_block;
 
@@ -114,25 +117,25 @@ struct msgpack_buffer_t {
     size_t write_reference_threshold;
     size_t read_reference_threshold;
     size_t io_buffer_size;
-
-    VALUE owner;
 };
 
 /*
  * initialization functions
  */
-void msgpack_buffer_static_init();
+void msgpack_buffer_static_init(void);
 
-void msgpack_buffer_static_destroy();
+void msgpack_buffer_static_destroy(void);
 
 void msgpack_buffer_init(msgpack_buffer_t* b);
 
 void msgpack_buffer_destroy(msgpack_buffer_t* b);
 
-void msgpack_buffer_mark(msgpack_buffer_t* b);
+void msgpack_buffer_mark(void* b);
 
 void msgpack_buffer_clear(msgpack_buffer_t* b);
 
+size_t msgpack_buffer_memsize(const msgpack_buffer_t* b);
+
 static inline void msgpack_buffer_set_write_reference_threshold(msgpack_buffer_t* b, size_t length)
 {
     if(length < MSGPACK_BUFFER_STRING_WRITE_REFERENCE_MINIMUM) {
@@ -265,14 +268,9 @@ static inline size_t msgpack_buffer_append_string(msgpack_buffer_t* b, VALUE str
 static inline size_t msgpack_buffer_append_string_reference(msgpack_buffer_t* b, VALUE string)
 {
     size_t length = RSTRING_LEN(string);
-
-    if(length > MSGPACK_BUFFER_STRING_WRITE_REFERENCE_MINIMUM) {
+    if (length > 0) {
         _msgpack_buffer_append_long_string(b, string);
-
-    } else {
-        msgpack_buffer_append(b, RSTRING_PTR(string), length);
     }
-
     return length;
 }
 
@@ -438,24 +436,65 @@ static inline VALUE _msgpack_buffer_refer_head_mapped_string(msgpack_buffer_t* b
     return rb_str_substr(b->head->mapped_string, offset, length);
 }
 
-static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_t length, bool will_be_frozen)
+static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_t length, bool will_be_frozen, bool utf8)
 {
-#ifndef DISABLE_BUFFER_READ_REFERENCE_OPTIMIZE
     /* optimize */
     if(!will_be_frozen &&
             b->head->mapped_string != NO_MAPPED_STRING &&
             length >= b->read_reference_threshold) {
         VALUE result = _msgpack_buffer_refer_head_mapped_string(b, length);
+        if (utf8) ENCODING_SET(result, msgpack_rb_encindex_utf8);
         _msgpack_buffer_consumed(b, length);
         return result;
     }
-#endif
 
-    VALUE result = rb_str_new(b->read_buffer, length);
+    VALUE result;
+
+#ifdef HAVE_RB_ENC_INTERNED_STR
+    if (will_be_frozen) {
+        if (RB_ENC_INTERNED_STR_NULL_CHECK && length == 0) {
+            result = rb_enc_interned_str("", length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        } else {
+            result = rb_enc_interned_str(b->read_buffer, length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        }
+    } else {
+        if (utf8) {
+            result = rb_utf8_str_new(b->read_buffer, length);
+        } else {
+            result = rb_str_new(b->read_buffer, length);
+        }
+    }
+    _msgpack_buffer_consumed(b, length);
+    return result;
+
+#else
+
+    if (utf8) {
+        result = rb_utf8_str_new(b->read_buffer, length);
+    } else {
+        result = rb_str_new(b->read_buffer, length);
+    }
+
+    if (will_be_frozen) {
+#if STR_UMINUS_DEDUPE_FROZEN
+        // Starting from MRI 2.8 it is preferable to freeze the string
+        // before deduplication so that it can be interned directly
+        // otherwise it would be duplicated first which is wasteful.
+        rb_str_freeze(result);
+#endif //STR_UMINUS_DEDUPE_FROZEN
+        // MRI 2.5 and older do not deduplicate strings that are already
+        // frozen.
+        result = rb_funcall(result, s_uminus, 0);
+    }
     _msgpack_buffer_consumed(b, length);
     return result;
+
+#endif // HAVE_RB_ENC_INTERNED_STR
 }
 
+static inline VALUE msgpack_buffer_read_top_as_symbol(msgpack_buffer_t* b, size_t length, bool utf8)
+{
+    return rb_str_intern(msgpack_buffer_read_top_as_string(b, length, true, utf8));
+}
 
 #endif
-