msgpack 1.5.1 → 1.7.2

data/ext/msgpack/buffer_class.c

@@ -21,25 +21,87 @@
 #include "buffer.h"
 #include "buffer_class.h"
 
-VALUE cMessagePack_Buffer;
+VALUE cMessagePack_Buffer = Qnil;
+VALUE cMessagePack_HeldBuffer = Qnil;
 
 static ID s_read;
 static ID s_readpartial;
 static ID s_write;
 static ID s_append;
 static ID s_close;
+static ID s_at_owner;
 
 static VALUE sym_read_reference_threshold;
 static VALUE sym_write_reference_threshold;
 static VALUE sym_io_buffer_size;
 
+typedef struct msgpack_held_buffer_t msgpack_held_buffer_t;
+struct msgpack_held_buffer_t {
+    size_t size;
+    VALUE mapped_strings[];
+};
 
-#define BUFFER(from, name) \
-    msgpack_buffer_t *name = NULL; \
-    Data_Get_Struct(from, msgpack_buffer_t, name); \
-    if(name == NULL) { \
-        rb_raise(rb_eArgError, "NULL found for " # name " when shouldn't be."); \
+static void HeldBuffer_mark(void *data)
+{
+    msgpack_held_buffer_t* held_buffer = (msgpack_held_buffer_t*)data;
+    for (size_t index = 0; index < held_buffer->size; index++) {
+        rb_gc_mark(held_buffer->mapped_strings[index]);
+    }
+}
+
+static size_t HeldBuffer_memsize(const void *data)
+{
+    const msgpack_held_buffer_t* held_buffer = (msgpack_held_buffer_t*)data;
+    return sizeof(size_t) + sizeof(VALUE) * held_buffer->size;
+}
+
+static const rb_data_type_t held_buffer_data_type = {
+    .wrap_struct_name = "msgpack:held_buffer",
+    .function = {
+        .dmark = HeldBuffer_mark,
+        .dfree = RUBY_TYPED_DEFAULT_FREE,
+        .dsize = HeldBuffer_memsize,
+    },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+VALUE MessagePack_Buffer_hold(msgpack_buffer_t* buffer)
+{
+    size_t mapped_strings_count = 0;
+    msgpack_buffer_chunk_t* c = buffer->head;
+    while (c != &buffer->tail) {
+        if (c->mapped_string != NO_MAPPED_STRING) {
+            mapped_strings_count++;
+        }
+        c = c->next;
+    }
+    if (c->mapped_string != NO_MAPPED_STRING) {
+        mapped_strings_count++;
+    }
+
+    if (mapped_strings_count == 0) {
+        return Qnil;
+    }
+
+    msgpack_held_buffer_t* held_buffer = xmalloc(sizeof(msgpack_held_buffer_t) + mapped_strings_count * sizeof(VALUE));
+
+    c = buffer->head;
+    mapped_strings_count = 0;
+    while (c != &buffer->tail) {
+        if (c->mapped_string != NO_MAPPED_STRING) {
+            held_buffer->mapped_strings[mapped_strings_count] = c->mapped_string;
+            mapped_strings_count++;
+        }
+        c = c->next;
     }
+    if (c->mapped_string != NO_MAPPED_STRING) {
+        held_buffer->mapped_strings[mapped_strings_count] = c->mapped_string;
+        mapped_strings_count++;
+    }
+    held_buffer->size = mapped_strings_count;
+    return TypedData_Wrap_Struct(cMessagePack_HeldBuffer, &held_buffer_data_type, held_buffer);
+}
+
 
 #define CHECK_STRING_TYPE(value) \
     value = rb_check_string_type(value); \
@@ -57,12 +119,49 @@ static void Buffer_free(void* data)
     xfree(b);
 }
 
+static size_t Buffer_memsize(const void *data)
+{
+    return sizeof(msgpack_buffer_t) + msgpack_buffer_memsize(data);
+}
+
+static const rb_data_type_t buffer_data_type = {
+    .wrap_struct_name = "msgpack:buffer",
+    .function = {
+        .dmark = msgpack_buffer_mark,
+        .dfree = Buffer_free,
+        .dsize = Buffer_memsize,
+    },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static const rb_data_type_t buffer_view_data_type = {
+    .wrap_struct_name = "msgpack:buffer_view",
+    .function = {
+        .dmark = NULL,
+        .dfree = NULL,
+        .dsize = NULL,
+    },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static inline msgpack_buffer_t *MessagePack_Buffer_get(VALUE object)
+{
+    msgpack_buffer_t *buffer;
+    bool view = RTEST(rb_ivar_get(object, s_at_owner));
+    TypedData_Get_Struct(object, msgpack_buffer_t, view ? &buffer_view_data_type : &buffer_data_type, buffer);
+    if (!buffer) {
+        rb_raise(rb_eArgError, "Uninitialized Buffer object");
+    }
+    return buffer;
+}
+
 static VALUE Buffer_alloc(VALUE klass)
 {
-    msgpack_buffer_t* b = ZALLOC_N(msgpack_buffer_t, 1);
+    msgpack_buffer_t* b;
+    VALUE buffer = TypedData_Make_Struct(klass, msgpack_buffer_t, &buffer_data_type, b);
     msgpack_buffer_init(b);
-
-    return Data_Wrap_Struct(klass, msgpack_buffer_mark, Buffer_free, b);
+    rb_ivar_set(buffer, s_at_owner, Qnil);
+    return buffer;
 }
 
 static ID get_partial_read_method(VALUE io)
@@ -96,25 +195,26 @@ void MessagePack_Buffer_set_options(msgpack_buffer_t* b, VALUE io, VALUE options
 
         v = rb_hash_aref(options, sym_read_reference_threshold);
         if(v != Qnil) {
-            msgpack_buffer_set_read_reference_threshold(b, NUM2ULONG(v));
+            msgpack_buffer_set_read_reference_threshold(b, NUM2SIZET(v));
         }
 
         v = rb_hash_aref(options, sym_write_reference_threshold);
         if(v != Qnil) {
-            msgpack_buffer_set_write_reference_threshold(b, NUM2ULONG(v));
+            msgpack_buffer_set_write_reference_threshold(b, NUM2SIZET(v));
         }
 
         v = rb_hash_aref(options, sym_io_buffer_size);
         if(v != Qnil) {
-            msgpack_buffer_set_io_buffer_size(b, NUM2ULONG(v));
+            msgpack_buffer_set_io_buffer_size(b, NUM2SIZET(v));
         }
     }
 }
 
 VALUE MessagePack_Buffer_wrap(msgpack_buffer_t* b, VALUE owner)
 {
-    b->owner = owner;
-    return Data_Wrap_Struct(cMessagePack_Buffer, msgpack_buffer_mark, NULL, b);
+    VALUE buffer = TypedData_Wrap_Struct(cMessagePack_Buffer, &buffer_view_data_type, b);
+    rb_ivar_set(buffer, s_at_owner, owner);
+    return buffer;
 }
 
 static VALUE Buffer_initialize(int argc, VALUE* argv, VALUE self)
@@ -144,7 +244,7 @@ static VALUE Buffer_initialize(int argc, VALUE* argv, VALUE self)
         rb_raise(rb_eArgError, "wrong number of arguments (%d for 0..1)", argc);
     }
 
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     MessagePack_Buffer_set_options(b, io, options);
 
@@ -153,21 +253,21 @@ static VALUE Buffer_initialize(int argc, VALUE* argv, VALUE self)
 
 static VALUE Buffer_clear(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     msgpack_buffer_clear(b);
     return Qnil;
 }
 
 static VALUE Buffer_size(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     size_t size = msgpack_buffer_all_readable_size(b);
     return SIZET2NUM(size);
 }
 
 static VALUE Buffer_empty_p(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     if(msgpack_buffer_top_readable_size(b) == 0) {
         return Qtrue;
     } else {
@@ -177,7 +277,7 @@ static VALUE Buffer_empty_p(VALUE self)
 
 static VALUE Buffer_write(VALUE self, VALUE string_or_buffer)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     VALUE string = string_or_buffer;  // TODO optimize if string_or_buffer is a Buffer
     StringValue(string);
@@ -189,7 +289,7 @@ static VALUE Buffer_write(VALUE self, VALUE string_or_buffer)
 
 static VALUE Buffer_append(VALUE self, VALUE string_or_buffer)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     VALUE string = string_or_buffer;  // TODO optimize if string_or_buffer is a Buffer
     StringValue(string);
@@ -280,14 +380,13 @@ static inline size_t read_until_eof(msgpack_buffer_t* b, VALUE out, unsigned lon
 
 static inline VALUE read_all(msgpack_buffer_t* b, VALUE out)
 {
-#ifndef DISABLE_BUFFER_READ_TO_S_OPTIMIZE
     if(out == Qnil && !msgpack_buffer_has_io(b)) {
         /* same as to_s && clear; optimize */
         VALUE str = msgpack_buffer_all_as_string(b);
         msgpack_buffer_clear(b);
         return str;
     }
-#endif
+
     MAKE_EMPTY_STRING(out);
     read_until_eof(b, out, 0);
     return out;
@@ -295,22 +394,22 @@ static inline VALUE read_all(msgpack_buffer_t* b, VALUE out)
 
 static VALUE Buffer_skip(VALUE self, VALUE sn)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     unsigned long n = FIX2ULONG(sn);
 
     /* do nothing */
     if(n == 0) {
-        return ULONG2NUM(0);
+        return INT2NUM(0);
     }
 
     size_t sz = read_until_eof(b, Qnil, n);
-    return ULONG2NUM(sz);
+    return SIZET2NUM(sz);
 }
 
 static VALUE Buffer_skip_all(VALUE self, VALUE sn)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     unsigned long n = FIX2ULONG(sn);
 
@@ -348,7 +447,7 @@ static VALUE Buffer_read_all(int argc, VALUE* argv, VALUE self)
         rb_raise(rb_eArgError, "wrong number of arguments (%d for 0..2)", argc);
     }
 
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     if(out != Qnil) {
         CHECK_STRING_TYPE(out);
@@ -394,7 +493,7 @@ static VALUE Buffer_read(int argc, VALUE* argv, VALUE self)
         rb_raise(rb_eArgError, "wrong number of arguments (%d for 0..2)", argc);
     }
 
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
 
     if(out != Qnil) {
         CHECK_STRING_TYPE(out);
@@ -410,7 +509,6 @@ static VALUE Buffer_read(int argc, VALUE* argv, VALUE self)
         return out;
     }
 
-#ifndef DISABLE_BUFFER_READ_TO_S_OPTIMIZE
     if(!msgpack_buffer_has_io(b) && out == Qnil &&
             msgpack_buffer_all_readable_size(b) <= n) {
         /* same as to_s && clear; optimize */
@@ -423,7 +521,6 @@ static VALUE Buffer_read(int argc, VALUE* argv, VALUE self)
             return str;
         }
     }
-#endif
 
     MAKE_EMPTY_STRING(out);
     read_until_eof(b, out, n);
@@ -437,32 +534,32 @@ static VALUE Buffer_read(int argc, VALUE* argv, VALUE self)
 
 static VALUE Buffer_to_str(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     return msgpack_buffer_all_as_string(b);
 }
 
 static VALUE Buffer_to_a(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     return msgpack_buffer_all_as_string_array(b);
 }
 
 static VALUE Buffer_flush(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     msgpack_buffer_flush(b);
     return self;
 }
 
 static VALUE Buffer_io(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     return b->io;
 }
 
 static VALUE Buffer_close(VALUE self)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     if(b->io != Qnil) {
         return rb_funcall(b->io, s_close, 0);
     }
@@ -471,9 +568,9 @@ static VALUE Buffer_close(VALUE self)
 
 static VALUE Buffer_write_to(VALUE self, VALUE io)
 {
-    BUFFER(self, b);
+    msgpack_buffer_t *b = MessagePack_Buffer_get(self);
     size_t sz = msgpack_buffer_flush_to_io(b, io, s_write, true);
-    return ULONG2NUM(sz);
+    return SIZET2NUM(sz);
 }
 
 void MessagePack_Buffer_module_init(VALUE mMessagePack)
@@ -483,6 +580,7 @@ void MessagePack_Buffer_module_init(VALUE mMessagePack)
     s_write = rb_intern("write");
     s_append = rb_intern("<<");
     s_close = rb_intern("close");
+    s_at_owner = rb_intern("@owner");
 
     sym_read_reference_threshold = ID2SYM(rb_intern("read_reference_threshold"));
     sym_write_reference_threshold = ID2SYM(rb_intern("write_reference_threshold"));
@@ -490,6 +588,9 @@ void MessagePack_Buffer_module_init(VALUE mMessagePack)
 
     msgpack_buffer_static_init();
 
+    cMessagePack_HeldBuffer = rb_define_class_under(mMessagePack, "HeldBuffer", rb_cBasicObject);
+    rb_undef_alloc_func(cMessagePack_HeldBuffer);
+
     cMessagePack_Buffer = rb_define_class_under(mMessagePack, "Buffer", rb_cObject);
 
     rb_define_alloc_func(cMessagePack_Buffer, Buffer_alloc);

data/ext/msgpack/buffer_class.h

@@ -25,6 +25,7 @@ extern VALUE cMessagePack_Buffer;
 void MessagePack_Buffer_module_init(VALUE mMessagePack);
 
 VALUE MessagePack_Buffer_wrap(msgpack_buffer_t* b, VALUE owner);
+VALUE MessagePack_Buffer_hold(msgpack_buffer_t* b);
 
 void MessagePack_Buffer_set_options(msgpack_buffer_t* b, VALUE io, VALUE options);
 

data/ext/msgpack/compat.h

@@ -22,104 +22,5 @@
 #include "ruby.h"
 #include "ruby/encoding.h"
 
-#if defined(HAVE_RUBY_ST_H)
-#  include "ruby/st.h"  /* ruby hash on Ruby 1.9 */
-#elif defined(HAVE_ST_H)
-#  include "st.h"       /* ruby hash on Ruby 1.8 */
-#endif
-
-
-/*
- * ZALLOC_N (ruby 2.2 or later)
- */
-#ifndef RB_ZALLOC_N
-#  define RB_ZALLOC_N(type,n) ((type*)ruby_xcalloc((size_t)(n),sizeof(type)))
-#endif
-#ifndef ZALLOC_N
-#  define ZALLOC_N(type,n) RB_ZALLOC_N(type,n)
-#endif
-
-
-/*
- * define STR_DUP_LIKELY_DOES_COPY
- * check rb_str_dup actually copies the string or not
- */
-#if defined(RUBY_VM) && defined(FL_ALL) && defined(FL_USER1) && defined(FL_USER3)  /* MRI 1.9 */
-#  define STR_DUP_LIKELY_DOES_COPY(str) FL_ALL(str, FL_USER1|FL_USER3)  /* same as STR_ASSOC_P(str) */
-
-#elif defined(FL_TEST) && defined(ELTS_SHARED)  /* MRI 1.8 */
-#  define STR_DUP_LIKELY_DOES_COPY(str) (!FL_TEST(str, ELTS_SHARED))
-
-//#elif defined(RUBINIUS) || defined(JRUBY)  /* Rubinius and JRuby */
-#else
-#  define STR_DUP_LIKELY_DOES_COPY(str) (1)
-
-#endif
-
-
-/*
- * SIZET2NUM
- */
-#ifndef SIZET2NUM   /* MRI 1.8 */
-#  define SIZET2NUM(v) ULL2NUM(v)
-#endif
-
-
-/*
- * rb_errinfo()
- */
-#if defined(RUBY_VM)  /* MRI 1.9 */
-#  define COMPAT_RERAISE rb_exc_raise(rb_errinfo())
-
-#elif defined(JRUBY)  /* JRuby */
-#  define COMPAT_RERAISE rb_exc_raise(rb_gv_get("$!"))
-
-#else  /* MRI 1.8 and Rubinius */
-#  define COMPAT_RERAISE rb_exc_raise(ruby_errinfo)
-#endif
-
-
-/*
- * RBIGNUM_POSITIVE_P
- */
-#ifndef RBIGNUM_POSITIVE_P
-#  if defined(RUBINIUS)  /* Rubinius <= v1.2.3 */
-#    define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue)
-
-#  elif defined(JRUBY)  /* JRuby */
-#    define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue)
-#    define rb_big2ull(b) rb_num2ull(b)
-     /*#define rb_big2ll(b) rb_num2ll(b)*/
-
-#  else  /* MRI 1.8 */
-#    define RBIGNUM_POSITIVE_P(b) (RBIGNUM(b)->sign)
-#  endif
-#endif
-
-
-/*
- * RSTRING_PTR, RSTRING_LEN
- */
-#ifndef RSTRING_PTR  /* MRI 1.8.5 */
-#  define RSTRING_PTR(s) (RSTRING(s)->ptr)
-#endif
-
-#ifndef RSTRING_LEN  /* MRI 1.8.5 */
-#  define RSTRING_LEN(s) (RSTRING(s)->len)
-#endif
-
-
-/*
- * RSTRUCT_GET
- */
-#ifndef RSTRUCT_GET
-#  ifdef RSTRUCT_PTR  /* MRI <= 2.0.0 */
-#    define RSTRUCT_GET(st, idx)  (RSTRUCT_PTR(st)[idx])
-#  else /* Rubinius */
-#    define RSTRUCT_GET(st, idx)  (rb_struct_aref(st, INT2FIX(idx)))
-#  endif
-#endif
-
-
 #endif
 

data/ext/msgpack/extconf.rb

@@ -1,60 +1,47 @@
 require 'mkmf'
 
-have_header("ruby/st.h")
-have_header("st.h")
-have_func("rb_enc_interned_str", "ruby.h")
-
-unless RUBY_PLATFORM.include? 'mswin'
-  $CFLAGS << %[ -I.. -Wall -O3 -g -std=gnu99]
-end
-#$CFLAGS << %[ -DDISABLE_RMEM]
-#$CFLAGS << %[ -DDISABLE_RMEM_REUSE_INTERNAL_FRAGMENT]
-#$CFLAGS << %[ -DDISABLE_BUFFER_READ_REFERENCE_OPTIMIZE]
-#$CFLAGS << %[ -DDISABLE_BUFFER_READ_TO_S_OPTIMIZE]
-
-if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'rbx'
-  # msgpack-ruby doesn't modify data came from RSTRING_PTR(str)
-  $CFLAGS << %[ -DRSTRING_NOT_MODIFIED]
-  # Rubinius C extensions don't grab GVL while rmem is not thread safe
-  $CFLAGS << %[ -DDISABLE_RMEM]
+have_func("rb_enc_interned_str", "ruby.h") # Ruby 3.0+
+have_func("rb_hash_new_capa", "ruby.h") # Ruby 3.2+
+have_func("rb_proc_call_with_block", "ruby.h") # CRuby (TruffleRuby doesn't have it)
+
+append_cflags([
+  "-fvisibility=hidden",
+  "-I..",
+  "-Wall",
+  "-O3",
+  "-std=gnu99"
+])
+append_cflags(RbConfig::CONFIG["debugflags"]) if RbConfig::CONFIG["debugflags"]
+
+append_cflags("-DRUBY_DEBUG=1") if ENV["MSGPACK_DEBUG"]
+
+if RUBY_VERSION.start_with?('3.0.') && RUBY_VERSION <= '3.0.5'
+  # https://bugs.ruby-lang.org/issues/18772
+  append_cflags("-DRB_ENC_INTERNED_STR_NULL_CHECK=1")
 end
 
-# checking if Hash#[]= (rb_hash_aset) dedupes string keys
+# checking if Hash#[]= (rb_hash_aset) dedupes string keys (Ruby 2.6+)
 h = {}
 x = {}
 r = rand.to_s
 h[%W(#{r}).join('')] = :foo
 x[%W(#{r}).join('')] = :foo
 if x.keys[0].equal?(h.keys[0])
-  $CFLAGS << ' -DHASH_ASET_DEDUPE=1 '
+  append_cflags("-DHASH_ASET_DEDUPE=1")
 else
-  $CFLAGS << ' -DHASH_ASET_DEDUPE=0 '
+  append_cflags("-DHASH_ASET_DEDUPE=0")
 end
 
-
-# checking if String#-@ (str_uminus) dedupes... '
-begin
-  a = -(%w(t e s t).join)
-  b = -(%w(t e s t).join)
-  if a.equal?(b)
-    $CFLAGS << ' -DSTR_UMINUS_DEDUPE=1 '
-  else
-    $CFLAGS += ' -DSTR_UMINUS_DEDUPE=0 '
-  end
-rescue NoMethodError
-  $CFLAGS << ' -DSTR_UMINUS_DEDUPE=0 '
-end
-
-# checking if String#-@ (str_uminus) directly interns frozen strings... '
+# checking if String#-@ (str_uminus) directly interns frozen strings... ' (Ruby 3.0+)
 begin
   s = rand.to_s.freeze
   if (-s).equal?(s) && (-s.dup).equal?(s)
-    $CFLAGS << ' -DSTR_UMINUS_DEDUPE_FROZEN=1 '
+    append_cflags("-DSTR_UMINUS_DEDUPE_FROZEN=1")
   else
-    $CFLAGS << ' -DSTR_UMINUS_DEDUPE_FROZEN=0 '
+    append_cflags("-DSTR_UMINUS_DEDUPE_FROZEN=0")
   end
 rescue NoMethodError
-  $CFLAGS << ' -DSTR_UMINUS_DEDUPE_FROZEN=0 '
+  append_cflags("-DSTR_UMINUS_DEDUPE_FROZEN=0")
 end
 
 if warnflags = CONFIG['warnflags']
@@ -62,4 +49,3 @@ if warnflags = CONFIG['warnflags']
 end
 
 create_makefile('msgpack/msgpack')
-