msgpack 1.5.2 → 1.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7de0658b1f87162270c3cc2aac70ffff8d408d974faac6ff5ad009fa11dba4dd
-  data.tar.gz: 9da35f2cc47ced45efd96b10ad8a3b491f0658b5ba247ad7b4aa22b901c66110
+  metadata.gz: e2f5a7cb55341aa8e930c92374b3aeb7bc74b1ccdfe0dede04e36169ce63354f
+  data.tar.gz: 42d92aff731b1a0ba35d775390c381bcff95fdb25ad8993248ca61753fb4b9e3
 SHA512:
-  metadata.gz: b23c683b4c7fbbedd8dedc5d6eb8f5d6778aa3d3f7afef839cfe3227fc2edff713d9f17d6c8ce3808a6103325d7923ca6abfaf8792dba11485ed9d067fe6aa42
-  data.tar.gz: ced82946481b00d99a69aea792efbeb93982d39836bc763c7c4a0f71844ca48044cff1f58c9695aa4ebb19d0be6a0b43f2e3b4af04ca06d7ad4bfc798cacd0b8
+  metadata.gz: 3cae7bc28afab9c63303d2aab6eb66e892fdacb0a5ba21113d7d4f8deac7647a18e2f2e708c66764671e78221a04b388c0bda93a3c8b2d2732c4b66d805408c5
+  data.tar.gz: d7fb26c1078340d7eed9d3589500c0fd4fcbb3aa995af82fbe632167494bd2400c9247f3d8c8b5091a3b0ad0e77a9f2624a154adc1781dbf611d6dc50f682afe

data/ChangeLog

@@ -1,3 +1,17 @@
+2022-08-22 1.5.5:
+
+* Fix a segfault when GC triggers inside a recursive extension.
+
+2022-07-25 1.5.4:
+
+* Fix a segfault when deserializing empty symbol (`:""`).
+* Improve compilation flags to not strip debug symbols.
+
+2022-05-30 version 1.5.3:
+
+* Fix deduplication of empty strings when using the `freeze: true` option.
+* Use `rb_hash_new_capa` when available (Ruby 3.2) for improved performance when parsing large hashes.
+
 2022-05-27 version 1.5.2:
 
 * Fix bug about unpacking ext type objects with the recursive option

data/README.md

@@ -40,7 +40,7 @@ or build msgpack-ruby and install:
 MessagePack for Ruby should run on x86, ARM, PowerPC, SPARC and other CPU architectures.
 
 And it works with MRI (CRuby) and Rubinius.
-Patches to improve portability is highly welcomed.
+Patches to improve portability are highly welcomed.
 
 
 ## Serializing objects
@@ -51,6 +51,7 @@ Use `MessagePack.pack` or `to_msgpack`:
 require 'msgpack'
 msg = MessagePack.pack(obj)  # or
 msg = obj.to_msgpack
+File.binwrite('mydata.msgpack', msg)
 ```
 
 ### Streaming serialization
@@ -71,6 +72,7 @@ Use `MessagePack.unpack`:
 
 ```ruby
 require 'msgpack'
+msg = File.binread('mydata.msgpack')
 obj = MessagePack.unpack(msg)
 ```
 

data/bench/bench.rb

@@ -0,0 +1,78 @@
+# % bundle install
+# % bundle exec ruby bench/bench.rb
+
+require 'msgpack'
+
+require 'benchmark/ips'
+
+object_plain = {
+  'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"'
+}
+
+data_plain = MessagePack.pack(object_plain)
+
+object_structured = {
+  'remote_host' => '127.0.0.1',
+  'remote_user' => '-',
+  'date' => '10/Oct/2000:13:55:36 -0700',
+  'request' => 'GET /apache_pb.gif HTTP/1.0',
+  'method' => 'GET',
+  'path' => '/apache_pb.gif',
+  'protocol' => 'HTTP/1.0',
+  'status' => 200,
+  'bytes' => 2326,
+  'referer' => 'http://www.example.com/start.html',
+  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
+}
+
+data_structured = MessagePack.pack(object_structured)
+
+class Extended
+  def to_msgpack_ext
+    MessagePack.pack({})
+  end
+
+  def self.from_msgpack_ext(data)
+    MessagePack.unpack(data)
+    Extended.new
+  end
+end
+
+object_extended = {
+  'extended' => Extended.new
+}
+
+extended_packer = MessagePack::Packer.new
+extended_packer.register_type(0x00, Extended, :to_msgpack_ext)
+data_extended = extended_packer.pack(object_extended).to_s
+
+Benchmark.ips do |x|
+  x.report('pack-plain') do
+    MessagePack.pack(object_plain)
+  end
+
+  x.report('pack-structured') do
+    MessagePack.pack(object_structured)
+  end
+
+  x.report('pack-extended') do
+    packer = MessagePack::Packer.new
+    packer.register_type(0x00, Extended, :to_msgpack_ext)
+    packer.pack(object_extended).to_s
+  end
+
+  x.report('unpack-plain') do
+    MessagePack.unpack(data_plain)
+  end
+
+  x.report('unpack-structured') do
+    MessagePack.unpack(data_structured)
+  end
+
+  x.report('unpack-extended') do
+    unpacker = MessagePack::Unpacker.new
+    unpacker.register_type(0x00, Extended, :from_msgpack_ext)
+    unpacker.feed data_extended
+    unpacker.read
+  end
+end

data/ext/msgpack/buffer.c

@@ -300,30 +300,6 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
     }
 }
 
-static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE string)
-{
-    VALUE mapped_string = rb_str_dup(string);
-    ENCODING_SET(mapped_string, msgpack_rb_encindex_ascii8bit);
-
-    _msgpack_buffer_add_new_chunk(b);
-
-    char* data = RSTRING_PTR(mapped_string);
-    size_t length = RSTRING_LEN(mapped_string);
-
-    b->tail.first = (char*) data;
-    b->tail.last = (char*) data + length;
-    b->tail.mapped_string = mapped_string;
-    b->tail.mem = NULL;
-
-    /* msgpack_buffer_writable_size should return 0 for mapped chunk */
-    b->tail_buffer_end = b->tail.last;
-
-    /* consider read_buffer */
-    if(b->head == &b->tail) {
-        b->read_buffer = b->tail.first;
-    }
-}
-
 void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
 {
     size_t length = RSTRING_LEN(string);
@@ -332,16 +308,9 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
         msgpack_buffer_flush(b);
         if (ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit) {
             rb_funcall(b->io, b->io_write_all_method, 1, string);
-        } else if(!STR_DUP_LIKELY_DOES_COPY(string)) {
-            VALUE s = rb_str_dup(string);
-            ENCODING_SET(s, msgpack_rb_encindex_ascii8bit);
-            rb_funcall(b->io, b->io_write_all_method, 1, s);
         } else {
             msgpack_buffer_append(b, RSTRING_PTR(string), length);
         }
-    } else if(!STR_DUP_LIKELY_DOES_COPY(string)) {
-        _msgpack_buffer_append_reference(b, string);
-
     } else {
         msgpack_buffer_append(b, RSTRING_PTR(string), length);
     }
@@ -619,13 +588,13 @@ size_t msgpack_buffer_flush_to_io(msgpack_buffer_t* b, VALUE io, ID write_method
 size_t _msgpack_buffer_feed_from_io(msgpack_buffer_t* b)
 {
     if(b->io_buffer == Qnil) {
-        b->io_buffer = rb_funcall(b->io, b->io_partial_read_method, 1, LONG2NUM(b->io_buffer_size));
+        b->io_buffer = rb_funcall(b->io, b->io_partial_read_method, 1, SIZET2NUM(b->io_buffer_size));
         if(b->io_buffer == Qnil) {
             rb_raise(rb_eEOFError, "IO reached end of file");
         }
         StringValue(b->io_buffer);
     } else {
-        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(b->io_buffer_size), b->io_buffer);
+        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(b->io_buffer_size), b->io_buffer);
         if(ret == Qnil) {
             rb_raise(rb_eEOFError, "IO reached end of file");
         }
@@ -644,9 +613,11 @@ size_t _msgpack_buffer_feed_from_io(msgpack_buffer_t* b)
 
 size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string, size_t length)
 {
+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
+
     if(RSTRING_LEN(string) == 0) {
         /* direct read */
-        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(length), string);
+        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(MIN(b->io_buffer_size, length)), string);
         if(ret == Qnil) {
             return 0;
         }
@@ -658,7 +629,7 @@ size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string,
         b->io_buffer = rb_str_buf_new(0);
     }
 
-    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(length), b->io_buffer);
+    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(MIN(b->io_buffer_size, length)), b->io_buffer);
     if(ret == Qnil) {
         return 0;
     }
@@ -666,6 +637,8 @@ size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string,
 
     rb_str_buf_cat(string, (const void*)RSTRING_PTR(b->io_buffer), rl);
     return rl;
+
+#undef MIN
 }
 
 size_t _msgpack_buffer_skip_from_io(msgpack_buffer_t* b, size_t length)
@@ -674,7 +647,7 @@ size_t _msgpack_buffer_skip_from_io(msgpack_buffer_t* b, size_t length)
         b->io_buffer = rb_str_buf_new(0);
     }
 
-    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(length), b->io_buffer);
+    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(length), b->io_buffer);
     if(ret == Qnil) {
         return 0;
     }

data/ext/msgpack/buffer.h

@@ -49,6 +49,10 @@
 
 #define NO_MAPPED_STRING ((VALUE)0)
 
+#ifndef RB_ENC_INTERNED_STR_NULL_CHECK
+#define RB_ENC_INTERNED_STR_NULL_CHECK 0
+#endif
+
 extern int msgpack_rb_encindex_utf8;
 extern int msgpack_rb_encindex_usascii;
 extern int msgpack_rb_encindex_ascii8bit;
@@ -456,7 +460,11 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
 
 #ifdef HAVE_RB_ENC_INTERNED_STR
     if (will_be_frozen) {
-        result = rb_enc_interned_str(b->read_buffer, length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        if (RB_ENC_INTERNED_STR_NULL_CHECK && length == 0) {
+            result = rb_enc_interned_str("", length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        } else {
+            result = rb_enc_interned_str(b->read_buffer, length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        }
     } else {
         if (utf8) {
             result = rb_utf8_str_new(b->read_buffer, length);

data/ext/msgpack/buffer_class.c

@@ -96,17 +96,17 @@ void MessagePack_Buffer_set_options(msgpack_buffer_t* b, VALUE io, VALUE options
 
         v = rb_hash_aref(options, sym_read_reference_threshold);
         if(v != Qnil) {
-            msgpack_buffer_set_read_reference_threshold(b, NUM2ULONG(v));
+            msgpack_buffer_set_read_reference_threshold(b, NUM2SIZET(v));
         }
 
         v = rb_hash_aref(options, sym_write_reference_threshold);
         if(v != Qnil) {
-            msgpack_buffer_set_write_reference_threshold(b, NUM2ULONG(v));
+            msgpack_buffer_set_write_reference_threshold(b, NUM2SIZET(v));
         }
 
         v = rb_hash_aref(options, sym_io_buffer_size);
         if(v != Qnil) {
-            msgpack_buffer_set_io_buffer_size(b, NUM2ULONG(v));
+            msgpack_buffer_set_io_buffer_size(b, NUM2SIZET(v));
         }
     }
 }
@@ -301,11 +301,11 @@ static VALUE Buffer_skip(VALUE self, VALUE sn)
 
     /* do nothing */
     if(n == 0) {
-        return ULONG2NUM(0);
+        return INT2NUM(0);
     }
 
     size_t sz = read_until_eof(b, Qnil, n);
-    return ULONG2NUM(sz);
+    return SIZET2NUM(sz);
 }
 
 static VALUE Buffer_skip_all(VALUE self, VALUE sn)
@@ -473,7 +473,7 @@ static VALUE Buffer_write_to(VALUE self, VALUE io)
 {
     BUFFER(self, b);
     size_t sz = msgpack_buffer_flush_to_io(b, io, s_write, true);
-    return ULONG2NUM(sz);
+    return SIZET2NUM(sz);
 }
 
 void MessagePack_Buffer_module_init(VALUE mMessagePack)

data/ext/msgpack/compat.h

@@ -22,104 +22,5 @@
 #include "ruby.h"
 #include "ruby/encoding.h"
 
-#if defined(HAVE_RUBY_ST_H)
-#  include "ruby/st.h"  /* ruby hash on Ruby 1.9 */
-#elif defined(HAVE_ST_H)
-#  include "st.h"       /* ruby hash on Ruby 1.8 */
-#endif
-
-
-/*
- * ZALLOC_N (ruby 2.2 or later)
- */
-#ifndef RB_ZALLOC_N
-#  define RB_ZALLOC_N(type,n) ((type*)ruby_xcalloc((size_t)(n),sizeof(type)))
-#endif
-#ifndef ZALLOC_N
-#  define ZALLOC_N(type,n) RB_ZALLOC_N(type,n)
-#endif
-
-
-/*
- * define STR_DUP_LIKELY_DOES_COPY
- * check rb_str_dup actually copies the string or not
- */
-#if defined(RUBY_VM) && defined(FL_ALL) && defined(FL_USER1) && defined(FL_USER3)  /* MRI 1.9 */
-#  define STR_DUP_LIKELY_DOES_COPY(str) FL_ALL(str, FL_USER1|FL_USER3)  /* same as STR_ASSOC_P(str) */
-
-#elif defined(FL_TEST) && defined(ELTS_SHARED)  /* MRI 1.8 */
-#  define STR_DUP_LIKELY_DOES_COPY(str) (!FL_TEST(str, ELTS_SHARED))
-
-//#elif defined(RUBINIUS) || defined(JRUBY)  /* Rubinius and JRuby */
-#else
-#  define STR_DUP_LIKELY_DOES_COPY(str) (1)
-
-#endif
-
-
-/*
- * SIZET2NUM
- */
-#ifndef SIZET2NUM   /* MRI 1.8 */
-#  define SIZET2NUM(v) ULL2NUM(v)
-#endif
-
-
-/*
- * rb_errinfo()
- */
-#if defined(RUBY_VM)  /* MRI 1.9 */
-#  define COMPAT_RERAISE rb_exc_raise(rb_errinfo())
-
-#elif defined(JRUBY)  /* JRuby */
-#  define COMPAT_RERAISE rb_exc_raise(rb_gv_get("$!"))
-
-#else  /* MRI 1.8 and Rubinius */
-#  define COMPAT_RERAISE rb_exc_raise(ruby_errinfo)
-#endif
-
-
-/*
- * RBIGNUM_POSITIVE_P
- */
-#ifndef RBIGNUM_POSITIVE_P
-#  if defined(RUBINIUS)  /* Rubinius <= v1.2.3 */
-#    define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue)
-
-#  elif defined(JRUBY)  /* JRuby */
-#    define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue)
-#    define rb_big2ull(b) rb_num2ull(b)
-     /*#define rb_big2ll(b) rb_num2ll(b)*/
-
-#  else  /* MRI 1.8 */
-#    define RBIGNUM_POSITIVE_P(b) (RBIGNUM(b)->sign)
-#  endif
-#endif
-
-
-/*
- * RSTRING_PTR, RSTRING_LEN
- */
-#ifndef RSTRING_PTR  /* MRI 1.8.5 */
-#  define RSTRING_PTR(s) (RSTRING(s)->ptr)
-#endif
-
-#ifndef RSTRING_LEN  /* MRI 1.8.5 */
-#  define RSTRING_LEN(s) (RSTRING(s)->len)
-#endif
-
-
-/*
- * RSTRUCT_GET
- */
-#ifndef RSTRUCT_GET
-#  ifdef RSTRUCT_PTR  /* MRI <= 2.0.0 */
-#    define RSTRUCT_GET(st, idx)  (RSTRUCT_PTR(st)[idx])
-#  else /* Rubinius */
-#    define RSTRUCT_GET(st, idx)  (rb_struct_aref(st, INT2FIX(idx)))
-#  endif
-#endif
-
-
 #endif
 

data/ext/msgpack/extconf.rb

@@ -2,24 +2,23 @@ require 'mkmf'
 
 have_header("ruby/st.h")
 have_header("st.h")
-have_func("rb_enc_interned_str", "ruby.h")
+have_func("rb_enc_interned_str", "ruby.h") # Ruby 3.0+
+have_func("rb_hash_new_capa", "ruby.h") # Ruby 3.2+
 
 unless RUBY_PLATFORM.include? 'mswin'
-  $CFLAGS << %[ -I.. -Wall -O3 -g -std=gnu99]
+  $CFLAGS << %[ -I.. -Wall -O3 #{RbConfig::CONFIG["debugflags"]} -std=gnu99]
 end
 #$CFLAGS << %[ -DDISABLE_RMEM]
 #$CFLAGS << %[ -DDISABLE_RMEM_REUSE_INTERNAL_FRAGMENT]
 #$CFLAGS << %[ -DDISABLE_BUFFER_READ_REFERENCE_OPTIMIZE]
 #$CFLAGS << %[ -DDISABLE_BUFFER_READ_TO_S_OPTIMIZE]
 
-if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'rbx'
-  # msgpack-ruby doesn't modify data came from RSTRING_PTR(str)
-  $CFLAGS << %[ -DRSTRING_NOT_MODIFIED]
-  # Rubinius C extensions don't grab GVL while rmem is not thread safe
-  $CFLAGS << %[ -DDISABLE_RMEM]
+if RUBY_VERSION.start_with?('3.0.')
+  # https://bugs.ruby-lang.org/issues/18772
+  $CFLAGS << ' -DRB_ENC_INTERNED_STR_NULL_CHECK=1 '
 end
 
-# checking if Hash#[]= (rb_hash_aset) dedupes string keys
+# checking if Hash#[]= (rb_hash_aset) dedupes string keys (Ruby 2.6+)
 h = {}
 x = {}
 r = rand.to_s
@@ -32,7 +31,7 @@ else
 end
 
 
-# checking if String#-@ (str_uminus) dedupes... '
+# checking if String#-@ (str_uminus) dedupes... ' (Ruby 2.5+)
 begin
   a = -(%w(t e s t).join)
   b = -(%w(t e s t).join)
@@ -45,7 +44,7 @@ rescue NoMethodError
   $CFLAGS << ' -DSTR_UMINUS_DEDUPE=0 '
 end
 
-# checking if String#-@ (str_uminus) directly interns frozen strings... '
+# checking if String#-@ (str_uminus) directly interns frozen strings... ' (Ruby 3.0+)
 begin
   s = rand.to_s.freeze
   if (-s).equal?(s) && (-s.dup).equal?(s)
@@ -62,4 +61,3 @@ if warnflags = CONFIG['warnflags']
 end
 
 create_makefile('msgpack/msgpack')
-

data/ext/msgpack/packer.c

@@ -18,24 +18,10 @@
 
 #include "packer.h"
 
-#ifdef RUBINIUS
-static ID s_to_iter;
-static ID s_next;
-static ID s_key;
-static ID s_value;
-#endif
-
 static ID s_call;
 
 void msgpack_packer_static_init()
 {
-#ifdef RUBINIUS
-    s_to_iter = rb_intern("to_iter");
-    s_next = rb_intern("next");
-    s_key = rb_intern("key");
-    s_value = rb_intern("value");
-#endif
-
     s_call = rb_intern("call");
 }
 
@@ -108,17 +94,7 @@ void msgpack_packer_write_hash_value(msgpack_packer_t* pk, VALUE v)
     unsigned int len32 = (unsigned int)len;
     msgpack_packer_write_map_header(pk, len32);
 
-#ifdef RUBINIUS
-    VALUE iter = rb_funcall(v, s_to_iter, 0);
-    VALUE entry = Qnil;
-    while(RTEST(entry = rb_funcall(iter, s_next, 1, entry))) {
-        VALUE key = rb_funcall(entry, s_key, 0);
-        VALUE val = rb_funcall(entry, s_value, 0);
-        write_hash_foreach(key, val, (VALUE) pk);
-    }
-#else
     rb_hash_foreach(v, write_hash_foreach, (VALUE) pk);
-#endif
 }
 
 struct msgpack_call_proc_args_t;

data/ext/msgpack/packer.h

@@ -408,13 +408,7 @@ static inline bool msgpack_packer_is_utf8_compat_string(VALUE v, int encindex)
 {
     return encindex == msgpack_rb_encindex_utf8
         || encindex == msgpack_rb_encindex_usascii
-#ifdef ENC_CODERANGE_ASCIIONLY
-        /* Because ENC_CODERANGE_ASCIIONLY does not scan string, it may return ENC_CODERANGE_UNKNOWN unlike */
-        /* rb_enc_str_asciionly_p. It is always faster than rb_str_encode if it is available. */
-        /* Very old Rubinius (< v1.3.1) doesn't have ENC_CODERANGE_ASCIIONLY. */
-        || (rb_enc_asciicompat(rb_enc_from_index(encindex)) && ENC_CODERANGE_ASCIIONLY(v))
-#endif
-        ;
+        || (rb_enc_asciicompat(rb_enc_from_index(encindex)) && ENC_CODERANGE_ASCIIONLY(v));
 }
 
 static inline void msgpack_packer_write_string_value(msgpack_packer_t* pk, VALUE v)

data/ext/msgpack/packer_class.c

@@ -322,7 +322,7 @@ static VALUE Packer_write_to(VALUE self, VALUE io)
 {
     PACKER(self, pk);
     size_t sz = msgpack_buffer_flush_to_io(PACKER_BUFFER_(pk), io, s_write, true);
-    return ULONG2NUM(sz);
+    return SIZET2NUM(sz);
 }
 
 //static VALUE Packer_append(VALUE self, VALUE string_or_buffer)