msgpack 1.5.5 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2f5a7cb55341aa8e930c92374b3aeb7bc74b1ccdfe0dede04e36169ce63354f
-  data.tar.gz: 42d92aff731b1a0ba35d775390c381bcff95fdb25ad8993248ca61753fb4b9e3
+  metadata.gz: bb08f890b2d9a36312600e284933cd0de93a00a6b949507bfc3d7c5c7eab1638
+  data.tar.gz: 11e536f8bde329edbc966e5e1050134350024f89f1cb911878f86fd59ff1fc79
 SHA512:
-  metadata.gz: 3cae7bc28afab9c63303d2aab6eb66e892fdacb0a5ba21113d7d4f8deac7647a18e2f2e708c66764671e78221a04b388c0bda93a3c8b2d2732c4b66d805408c5
-  data.tar.gz: d7fb26c1078340d7eed9d3589500c0fd4fcbb3aa995af82fbe632167494bd2400c9247f3d8c8b5091a3b0ad0e77a9f2624a154adc1781dbf611d6dc50f682afe
+  metadata.gz: 9491cdf494d1826cce7f5e420d73a65ce88c029e060338d9c28fb1e20e4d01dc71fdb46b91859d025bbcc8f5a0471100b171cb9b3bac5fed6f0f0715fd76abb2
+  data.tar.gz: 8d0b4db4f93ebf98063f275c0629648ab9316c7cc3cd81738f7f083c6ed4c47fc647c2fc5df1308089b96c36d7e13944028e75bae7e5897b5bb357701c0bcfd9

data/ChangeLog

@@ -1,3 +1,15 @@
+2022-09-30 1.6.0:
+
+* Fix a potential use-after-free bug in Buffer_free when accessing a packer or unpacker buffer. 
+* `old-style-definition` compilation warnings.
+* Restore zero-copy buffer feed when provided a Ruby string. This was accidentally broken in 1.5.4.
+* Provide implementations for `ObjectSpace.memsize`. Message pack objects now properly report their size to Ruby.
+* Fix an endianess bug on Windows platform.
+
+2022-08-23 1.5.6:
+
+* No actual code change, just re-release the `java` version properly.
+
 2022-08-22 1.5.5:
 
 * Fix a segfault when GC triggers inside a recursive extension.

data/bin/console

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "msgpack"
+
+require "irb"
+IRB.start(__FILE__)

data/ext/msgpack/buffer.c

@@ -29,11 +29,9 @@ int msgpack_rb_encindex_ascii8bit;
 
 ID s_uminus;
 
-#ifndef DISABLE_RMEM
 static msgpack_rmem_t s_rmem;
-#endif
 
-void msgpack_buffer_static_init()
+void msgpack_buffer_static_init(void)
 {
     s_uminus = rb_intern("-@");
 
@@ -41,20 +39,16 @@ void msgpack_buffer_static_init()
     msgpack_rb_encindex_usascii = rb_usascii_encindex();
     msgpack_rb_encindex_ascii8bit = rb_ascii8bit_encindex();
 
-#ifndef DISABLE_RMEM
     msgpack_rmem_init(&s_rmem);
-#endif
 
 #ifndef HAVE_RB_STR_REPLACE
     s_replace = rb_intern("replace");
 #endif
 }
 
-void msgpack_buffer_static_destroy()
+void msgpack_buffer_static_destroy(void)
 {
-#ifndef DISABLE_RMEM
     msgpack_rmem_destroy(&s_rmem);
-#endif
 }
 
 void msgpack_buffer_init(msgpack_buffer_t* b)
@@ -72,16 +66,12 @@ void msgpack_buffer_init(msgpack_buffer_t* b)
 static void _msgpack_buffer_chunk_destroy(msgpack_buffer_chunk_t* c)
 {
     if(c->mem != NULL) {
-#ifndef DISABLE_RMEM
         if(!msgpack_rmem_free(&s_rmem, c->mem)) {
             xfree(c->mem);
         }
         /* no needs to update rmem_owner because chunks will not be
          * free()ed (left in free_list) and thus *rmem_owner is
          * always valid. */
-#else
-        xfree(c->mem);
-#endif
     }
     c->first = NULL;
     c->last = NULL;
@@ -108,8 +98,25 @@ void msgpack_buffer_destroy(msgpack_buffer_t* b)
     }
 }
 
-void msgpack_buffer_mark(msgpack_buffer_t* b)
+size_t msgpack_buffer_memsize(const msgpack_buffer_t* b)
+{
+    size_t memsize = 0;
+    msgpack_buffer_chunk_t* c = b->head;
+
+    while(c) {
+        memsize += sizeof(msgpack_buffer_chunk_t);
+        if(c->mapped_string != NO_MAPPED_STRING) {
+            memsize += (c->last - c->first);
+        }
+        c = c->next;
+    }
+
+    return memsize;
+}
+
+void msgpack_buffer_mark(void *ptr)
 {
+    msgpack_buffer_t* b = ptr;
     /* head is always available */
     msgpack_buffer_chunk_t* c = b->head;
     while(c != &b->tail) {
@@ -120,8 +127,6 @@ void msgpack_buffer_mark(msgpack_buffer_t* b)
 
     rb_gc_mark(b->io);
     rb_gc_mark(b->io_buffer);
-
-    rb_gc_mark(b->owner);
 }
 
 bool _msgpack_buffer_shift_chunk(msgpack_buffer_t* b)
@@ -158,7 +163,6 @@ size_t msgpack_buffer_read_to_string_nonblock(msgpack_buffer_t* b, VALUE string,
 {
     size_t avail = msgpack_buffer_top_readable_size(b);
 
-#ifndef DISABLE_BUFFER_READ_REFERENCE_OPTIMIZE
     /* optimize */
     if(length <= avail && RSTRING_LEN(string) == 0 &&
             b->head->mapped_string != NO_MAPPED_STRING &&
@@ -170,7 +174,6 @@ size_t msgpack_buffer_read_to_string_nonblock(msgpack_buffer_t* b, VALUE string,
         _msgpack_buffer_consumed(b, length);
         return length;
     }
-#endif
 
     size_t const length_orig = length;
 
@@ -283,15 +286,11 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
 
         msgpack_buffer_chunk_t* nc = _msgpack_buffer_alloc_new_chunk(b);
 
-#ifndef DISABLE_RMEM
-#ifndef DISABLE_RMEM_REUSE_INTERNAL_FRAGMENT
         if(b->rmem_last == b->tail_buffer_end) {
             /* reuse unused rmem space */
             size_t unused = b->tail_buffer_end - b->tail.last;
             b->rmem_last -= unused;
         }
-#endif
-#endif
 
         /* rebuild tail */
         *nc = b->tail;
@@ -300,6 +299,35 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
     }
 }
 
+static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE string)
+{
+    VALUE mapped_string;
+    if(ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit && RTEST(rb_obj_frozen_p(string))) {
+        mapped_string = string;
+    } else {
+        mapped_string = rb_str_dup(string);
+        ENCODING_SET(mapped_string, msgpack_rb_encindex_ascii8bit);
+    }
+
+    _msgpack_buffer_add_new_chunk(b);
+
+    char* data = RSTRING_PTR(mapped_string);
+    size_t length = RSTRING_LEN(mapped_string);
+
+    b->tail.first = (char*) data;
+    b->tail.last = (char*) data + length;
+    b->tail.mapped_string = mapped_string;
+    b->tail.mem = NULL;
+
+    /* msgpack_buffer_writable_size should return 0 for mapped chunk */
+    b->tail_buffer_end = b->tail.last;
+
+    /* consider read_buffer */
+    if(b->head == &b->tail) {
+        b->read_buffer = b->tail.first;
+    }
+}
+
 void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
 {
     size_t length = RSTRING_LEN(string);
@@ -312,7 +340,7 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
             msgpack_buffer_append(b, RSTRING_PTR(string), length);
         }
     } else {
-        msgpack_buffer_append(b, RSTRING_PTR(string), length);
+       _msgpack_buffer_append_reference(b, string);
     }
 }
 
@@ -320,11 +348,8 @@ static inline void* _msgpack_buffer_chunk_malloc(
         msgpack_buffer_t* b, msgpack_buffer_chunk_t* c,
         size_t required_size, size_t* allocated_size)
 {
-#ifndef DISABLE_RMEM
     if(required_size <= MSGPACK_RMEM_PAGE_SIZE) {
-#ifndef DISABLE_RMEM_REUSE_INTERNAL_FRAGMENT
         if((size_t)(b->rmem_end - b->rmem_last) < required_size) {
-#endif
             /* alloc new rmem page */
             *allocated_size = MSGPACK_RMEM_PAGE_SIZE;
             char* buffer = msgpack_rmem_alloc(&s_rmem);
@@ -335,8 +360,6 @@ static inline void* _msgpack_buffer_chunk_malloc(
             b->rmem_last = b->rmem_end = buffer + MSGPACK_RMEM_PAGE_SIZE;
 
             return buffer;
-
-#ifndef DISABLE_RMEM_REUSE_INTERNAL_FRAGMENT
         } else {
             /* reuse unused rmem */
             *allocated_size = (size_t)(b->rmem_end - b->rmem_last);
@@ -350,13 +373,7 @@ static inline void* _msgpack_buffer_chunk_malloc(
 
             return buffer;
         }
-#endif
-    }
-#else
-    if(required_size < 72) {
-        required_size = 72;
     }
-#endif
 
     // TODO alignment?
     *allocated_size = required_size;
@@ -411,11 +428,7 @@ void _msgpack_buffer_expand(msgpack_buffer_t* b, const char* data, size_t length
     size_t capacity = b->tail.last - b->tail.first;
 
     /* can't realloc mapped chunk or rmem page */
-    if(b->tail.mapped_string != NO_MAPPED_STRING
-#ifndef DISABLE_RMEM
-            || capacity <= MSGPACK_RMEM_PAGE_SIZE
-#endif
-            ) {
+    if(b->tail.mapped_string != NO_MAPPED_STRING || capacity <= MSGPACK_RMEM_PAGE_SIZE) {
         /* allocate new chunk */
         _msgpack_buffer_add_new_chunk(b);
 

data/ext/msgpack/buffer.h

@@ -102,11 +102,9 @@ struct msgpack_buffer_t {
     msgpack_buffer_chunk_t* head;
     msgpack_buffer_chunk_t* free_list;
 
-#ifndef DISABLE_RMEM
     char* rmem_last;
     char* rmem_end;
     void** rmem_owner;
-#endif
 
     union msgpack_buffer_cast_block_t cast_block;
 
@@ -118,25 +116,25 @@ struct msgpack_buffer_t {
     size_t write_reference_threshold;
     size_t read_reference_threshold;
     size_t io_buffer_size;
-
-    VALUE owner;
 };
 
 /*
  * initialization functions
  */
-void msgpack_buffer_static_init();
+void msgpack_buffer_static_init(void);
 
-void msgpack_buffer_static_destroy();
+void msgpack_buffer_static_destroy(void);
 
 void msgpack_buffer_init(msgpack_buffer_t* b);
 
 void msgpack_buffer_destroy(msgpack_buffer_t* b);
 
-void msgpack_buffer_mark(msgpack_buffer_t* b);
+void msgpack_buffer_mark(void* b);
 
 void msgpack_buffer_clear(msgpack_buffer_t* b);
 
+size_t msgpack_buffer_memsize(const msgpack_buffer_t* b);
+
 static inline void msgpack_buffer_set_write_reference_threshold(msgpack_buffer_t* b, size_t length)
 {
     if(length < MSGPACK_BUFFER_STRING_WRITE_REFERENCE_MINIMUM) {
@@ -444,7 +442,6 @@ static inline VALUE _msgpack_buffer_refer_head_mapped_string(msgpack_buffer_t* b
 
 static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_t length, bool will_be_frozen, bool utf8)
 {
-#ifndef DISABLE_BUFFER_READ_REFERENCE_OPTIMIZE
     /* optimize */
     if(!will_be_frozen &&
             b->head->mapped_string != NO_MAPPED_STRING &&
@@ -454,7 +451,6 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
         _msgpack_buffer_consumed(b, length);
         return result;
     }
-#endif
 
     VALUE result;
 

data/ext/msgpack/buffer_class.c

@@ -28,19 +28,13 @@ static ID s_readpartial;
 static ID s_write;
 static ID s_append;
 static ID s_close;
+static ID s_at_owner;
 
 static VALUE sym_read_reference_threshold;
 static VALUE sym_write_reference_threshold;
 static VALUE sym_io_buffer_size;
 
 
-#define BUFFER(from, name) \
-    msgpack_buffer_t *name = NULL; \
-    Data_Get_Struct(from, msgpack_buffer_t, name); \
-    if(name == NULL) { \
-        rb_raise(rb_eArgError, "NULL found for " # name " when shouldn't be."); \
-    }
-
 #define CHECK_STRING_TYPE(value) \
     value = rb_check_string_type(value); \
     if( NIL_P(value) ) { \
@@ -57,12 +51,49 @@ static void Buffer_free(void* data)
     xfree(b);
 }
 
+static size_t Buffer_memsize(const void *data)
+{
+    return sizeof(msgpack_buffer_t) + msgpack_buffer_memsize(data);
+}
+
+const rb_data_type_t buffer_data_type = {
+    .wrap_struct_name = "msgpack:buffer",
+    .function = {
+        .dmark = msgpack_buffer_mark,
+        .dfree = Buffer_free,
+        .dsize = Buffer_memsize,
+    },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+const rb_data_type_t buffer_view_data_type = {
+    .wrap_struct_name = "msgpack:buffer_view",
+    .function = {
+        .dmark = msgpack_buffer_mark,
+        .dfree = NULL,
+        .dsize = NULL,
+    },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static inline msgpack_buffer_t *MessagePack_Buffer_get(VALUE object)
+{
+    msgpack_buffer_t *buffer;
+    bool view = RTEST(rb_ivar_get(object, s_at_owner));
+    TypedData_Get_Struct(object, msgpack_buffer_t, view ? &buffer_view_data_type : &buffer_data_type, buffer);
+    if (!buffer) {
+        rb_raise(rb_eArgError, "Uninitialized Buffer object");
+    }
+    return buffer;
+}
+
 static VALUE Buffer_alloc(VALUE klass)
 {
-    msgpack_buffer_t* b = ZALLOC_N(msgpack_buffer_t, 1);
+    msgpack_buffer_t* b;
+    VALUE buffer = TypedData_Make_Struct(klass, msgpack_buffer_t, &buffer_data_type, b);
+    rb_ivar_set(buffer, s_at_owner, Qnil);
     msgpack_buffer_init(b);
-
-    return Data_Wrap_Struct(klass, msgpack_buffer_mark, Buffer_free, b);
+    return buffer;
 }
 
 static ID get_partial_read_method(VALUE io)
@@ -113,8 +144,9 @@ void MessagePack_Buffer_set_options(msgpack_buffer_t* b, VALUE io, VALUE options
 
 VALUE MessagePack_Buffer_wrap(msgpack_buffer_t* b, VALUE owner)
 {
-    b->owner = owner;
-    return Data_Wrap_Struct(cMessagePack_Buffer, msgpack_buffer_mark, NULL, b);
+    VALUE buffer = TypedData_Wrap_Struct(cMessagePack_Buffer, &buffer_view_data_type, b);
+    rb_ivar_set(buffer, s_at_owner, owner);
+    return buffer;
 }
 
 static VALUE Buffer_initialize(int argc, VALUE* argv, VALUE self)
@@ -144,7 +176,7 @@ static VALUE Buffer_initialize(int argc, VALUE* argv, VALUE self)
         rb_raise(rb_eArgError, "wrong number of arguments (%d for 0..1)", argc);
     }
 
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     MessagePack_Buffer_set_options(b, io, options);
 
@@ -153,21 +185,21 @@ static VALUE Buffer_initialize(int argc, VALUE* argv, VALUE self)
 
 static VALUE Buffer_clear(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     msgpack_buffer_clear(b);
     return Qnil;
 }
 
 static VALUE Buffer_size(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     size_t size = msgpack_buffer_all_readable_size(b);
     return SIZET2NUM(size);
 }
 
 static VALUE Buffer_empty_p(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     if(msgpack_buffer_top_readable_size(b) == 0) {
         return Qtrue;
     } else {
@@ -177,7 +209,7 @@ static VALUE Buffer_empty_p(VALUE self)
 
 static VALUE Buffer_write(VALUE self, VALUE string_or_buffer)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     VALUE string = string_or_buffer;  // TODO optimize if string_or_buffer is a Buffer
     StringValue(string);
@@ -189,7 +221,7 @@ static VALUE Buffer_write(VALUE self, VALUE string_or_buffer)
 
 static VALUE Buffer_append(VALUE self, VALUE string_or_buffer)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     VALUE string = string_or_buffer;  // TODO optimize if string_or_buffer is a Buffer
     StringValue(string);
@@ -280,14 +312,13 @@ static inline size_t read_until_eof(msgpack_buffer_t* b, VALUE out, unsigned lon
 
 static inline VALUE read_all(msgpack_buffer_t* b, VALUE out)
 {
-#ifndef DISABLE_BUFFER_READ_TO_S_OPTIMIZE
     if(out == Qnil && !msgpack_buffer_has_io(b)) {
         /* same as to_s && clear; optimize */
         VALUE str = msgpack_buffer_all_as_string(b);
         msgpack_buffer_clear(b);
         return str;
     }
-#endif
+
     MAKE_EMPTY_STRING(out);
     read_until_eof(b, out, 0);
     return out;
@@ -295,7 +326,7 @@ static inline VALUE read_all(msgpack_buffer_t* b, VALUE out)
 
 static VALUE Buffer_skip(VALUE self, VALUE sn)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     unsigned long n = FIX2ULONG(sn);
 
@@ -310,7 +341,7 @@ static VALUE Buffer_skip(VALUE self, VALUE sn)
 
 static VALUE Buffer_skip_all(VALUE self, VALUE sn)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     unsigned long n = FIX2ULONG(sn);
 
@@ -348,7 +379,7 @@ static VALUE Buffer_read_all(int argc, VALUE* argv, VALUE self)
         rb_raise(rb_eArgError, "wrong number of arguments (%d for 0..2)", argc);
     }
 
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     if(out != Qnil) {
         CHECK_STRING_TYPE(out);
@@ -394,7 +425,7 @@ static VALUE Buffer_read(int argc, VALUE* argv, VALUE self)
         rb_raise(rb_eArgError, "wrong number of arguments (%d for 0..2)", argc);
     }
 
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     if(out != Qnil) {
         CHECK_STRING_TYPE(out);
@@ -410,7 +441,6 @@ static VALUE Buffer_read(int argc, VALUE* argv, VALUE self)
         return out;
     }
 
-#ifndef DISABLE_BUFFER_READ_TO_S_OPTIMIZE
     if(!msgpack_buffer_has_io(b) && out == Qnil &&
             msgpack_buffer_all_readable_size(b) <= n) {
         /* same as to_s && clear; optimize */
@@ -423,7 +453,6 @@ static VALUE Buffer_read(int argc, VALUE* argv, VALUE self)
             return str;
         }
     }
-#endif
 
     MAKE_EMPTY_STRING(out);
     read_until_eof(b, out, n);
@@ -437,32 +466,32 @@ static VALUE Buffer_read(int argc, VALUE* argv, VALUE self)
 
 static VALUE Buffer_to_str(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     return msgpack_buffer_all_as_string(b);
 }
 
 static VALUE Buffer_to_a(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     return msgpack_buffer_all_as_string_array(b);
 }
 
 static VALUE Buffer_flush(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     msgpack_buffer_flush(b);
     return self;
 }
 
 static VALUE Buffer_io(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     return b->io;
 }
 
 static VALUE Buffer_close(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     if(b->io != Qnil) {
         return rb_funcall(b->io, s_close, 0);
     }
@@ -471,7 +500,7 @@ static VALUE Buffer_close(VALUE self)
 
 static VALUE Buffer_write_to(VALUE self, VALUE io)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     size_t sz = msgpack_buffer_flush_to_io(b, io, s_write, true);
     return SIZET2NUM(sz);
 }
@@ -483,6 +512,7 @@ void MessagePack_Buffer_module_init(VALUE mMessagePack)
     s_write = rb_intern("write");
     s_append = rb_intern("<<");
     s_close = rb_intern("close");
+    s_at_owner = rb_intern("@owner");
 
     sym_read_reference_threshold = ID2SYM(rb_intern("read_reference_threshold"));
     sym_write_reference_threshold = ID2SYM(rb_intern("write_reference_threshold"));

data/ext/msgpack/extconf.rb

@@ -8,10 +8,6 @@ have_func("rb_hash_new_capa", "ruby.h") # Ruby 3.2+
 unless RUBY_PLATFORM.include? 'mswin'
   $CFLAGS << %[ -I.. -Wall -O3 #{RbConfig::CONFIG["debugflags"]} -std=gnu99]
 end
-#$CFLAGS << %[ -DDISABLE_RMEM]
-#$CFLAGS << %[ -DDISABLE_RMEM_REUSE_INTERNAL_FRAGMENT]
-#$CFLAGS << %[ -DDISABLE_BUFFER_READ_REFERENCE_OPTIMIZE]
-#$CFLAGS << %[ -DDISABLE_BUFFER_READ_TO_S_OPTIMIZE]
 
 if RUBY_VERSION.start_with?('3.0.')
   # https://bugs.ruby-lang.org/issues/18772