msgpack 1.5.2 → 1.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7de0658b1f87162270c3cc2aac70ffff8d408d974faac6ff5ad009fa11dba4dd
-  data.tar.gz: 9da35f2cc47ced45efd96b10ad8a3b491f0658b5ba247ad7b4aa22b901c66110
+  metadata.gz: ec6ef147d4d948d86915dd5ed06893bd281bcba60a1a48869e0f1b895e1a4d7e
+  data.tar.gz: 49f4753d9ce5fc686c529a97b8fb409283cde4521c8e28a3f61d92fedc717148
 SHA512:
-  metadata.gz: b23c683b4c7fbbedd8dedc5d6eb8f5d6778aa3d3f7afef839cfe3227fc2edff713d9f17d6c8ce3808a6103325d7923ca6abfaf8792dba11485ed9d067fe6aa42
-  data.tar.gz: ced82946481b00d99a69aea792efbeb93982d39836bc763c7c4a0f71844ca48044cff1f58c9695aa4ebb19d0be6a0b43f2e3b4af04ca06d7ad4bfc798cacd0b8
+  metadata.gz: b094e3408d3d7411af91e81e3e59dfb23875696d2415b8759fa9d11476de795a98d7a2d1b9eee60a603157a3139614e5a349169c75d3737faabacbf4975bc31d
+  data.tar.gz: 27e8b222d6a20253b341e3dc5fe1f656e2698af47e95d2e1c40abb43c22a26c231cc680519e7464ba9b45745cd550f9a7ab649015311c69468f93944fd9dfc6a

data/ChangeLog

@@ -1,3 +1,13 @@
+2022-07-25
+
+* Fix a segfault when deserializing empty symbol (`:""`).
+* Improve compilation flags to not strip debug symbols.
+
+2022-05-30 version 1.5.3:
+
+* Fix deduplication of empty strings when using the `freeze: true` option.
+* Use `rb_hash_new_capa` when available (Ruby 3.2) for improved performance when parsing large hashes.
+
 2022-05-27 version 1.5.2:
 
 * Fix bug about unpacking ext type objects with the recursive option

data/README.md

@@ -40,7 +40,7 @@ or build msgpack-ruby and install:
 MessagePack for Ruby should run on x86, ARM, PowerPC, SPARC and other CPU architectures.
 
 And it works with MRI (CRuby) and Rubinius.
-Patches to improve portability is highly welcomed.
+Patches to improve portability are highly welcomed.
 
 
 ## Serializing objects
@@ -51,6 +51,7 @@ Use `MessagePack.pack` or `to_msgpack`:
 require 'msgpack'
 msg = MessagePack.pack(obj)  # or
 msg = obj.to_msgpack
+File.binwrite('mydata.msgpack', msg)
 ```
 
 ### Streaming serialization
@@ -71,6 +72,7 @@ Use `MessagePack.unpack`:
 
 ```ruby
 require 'msgpack'
+msg = File.binread('mydata.msgpack')
 obj = MessagePack.unpack(msg)
 ```
 

data/bench/bench.rb

@@ -0,0 +1,78 @@
+# % bundle install
+# % bundle exec ruby bench/bench.rb
+
+require 'msgpack'
+
+require 'benchmark/ips'
+
+object_plain = {
+  'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"'
+}
+
+data_plain = MessagePack.pack(object_plain)
+
+object_structured = {
+  'remote_host' => '127.0.0.1',
+  'remote_user' => '-',
+  'date' => '10/Oct/2000:13:55:36 -0700',
+  'request' => 'GET /apache_pb.gif HTTP/1.0',
+  'method' => 'GET',
+  'path' => '/apache_pb.gif',
+  'protocol' => 'HTTP/1.0',
+  'status' => 200,
+  'bytes' => 2326,
+  'referer' => 'http://www.example.com/start.html',
+  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
+}
+
+data_structured = MessagePack.pack(object_structured)
+
+class Extended
+  def to_msgpack_ext
+    MessagePack.pack({})
+  end
+
+  def self.from_msgpack_ext(data)
+    MessagePack.unpack(data)
+    Extended.new
+  end
+end
+
+object_extended = {
+  'extended' => Extended.new
+}
+
+extended_packer = MessagePack::Packer.new
+extended_packer.register_type(0x00, Extended, :to_msgpack_ext)
+data_extended = extended_packer.pack(object_extended).to_s
+
+Benchmark.ips do |x|
+  x.report('pack-plain') do
+    MessagePack.pack(object_plain)
+  end
+
+  x.report('pack-structured') do
+    MessagePack.pack(object_structured)
+  end
+
+  x.report('pack-extended') do
+    packer = MessagePack::Packer.new
+    packer.register_type(0x00, Extended, :to_msgpack_ext)
+    packer.pack(object_extended).to_s
+  end
+
+  x.report('unpack-plain') do
+    MessagePack.unpack(data_plain)
+  end
+
+  x.report('unpack-structured') do
+    MessagePack.unpack(data_structured)
+  end
+
+  x.report('unpack-extended') do
+    unpacker = MessagePack::Unpacker.new
+    unpacker.register_type(0x00, Extended, :from_msgpack_ext)
+    unpacker.feed data_extended
+    unpacker.read
+  end
+end

data/ext/msgpack/buffer.c

@@ -300,30 +300,6 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
     }
 }
 
-static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE string)
-{
-    VALUE mapped_string = rb_str_dup(string);
-    ENCODING_SET(mapped_string, msgpack_rb_encindex_ascii8bit);
-
-    _msgpack_buffer_add_new_chunk(b);
-
-    char* data = RSTRING_PTR(mapped_string);
-    size_t length = RSTRING_LEN(mapped_string);
-
-    b->tail.first = (char*) data;
-    b->tail.last = (char*) data + length;
-    b->tail.mapped_string = mapped_string;
-    b->tail.mem = NULL;
-
-    /* msgpack_buffer_writable_size should return 0 for mapped chunk */
-    b->tail_buffer_end = b->tail.last;
-
-    /* consider read_buffer */
-    if(b->head == &b->tail) {
-        b->read_buffer = b->tail.first;
-    }
-}
-
 void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
 {
     size_t length = RSTRING_LEN(string);
@@ -332,16 +308,9 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
         msgpack_buffer_flush(b);
         if (ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit) {
             rb_funcall(b->io, b->io_write_all_method, 1, string);
-        } else if(!STR_DUP_LIKELY_DOES_COPY(string)) {
-            VALUE s = rb_str_dup(string);
-            ENCODING_SET(s, msgpack_rb_encindex_ascii8bit);
-            rb_funcall(b->io, b->io_write_all_method, 1, s);
         } else {
             msgpack_buffer_append(b, RSTRING_PTR(string), length);
         }
-    } else if(!STR_DUP_LIKELY_DOES_COPY(string)) {
-        _msgpack_buffer_append_reference(b, string);
-
     } else {
         msgpack_buffer_append(b, RSTRING_PTR(string), length);
     }
@@ -619,13 +588,13 @@ size_t msgpack_buffer_flush_to_io(msgpack_buffer_t* b, VALUE io, ID write_method
 size_t _msgpack_buffer_feed_from_io(msgpack_buffer_t* b)
 {
     if(b->io_buffer == Qnil) {
-        b->io_buffer = rb_funcall(b->io, b->io_partial_read_method, 1, LONG2NUM(b->io_buffer_size));
+        b->io_buffer = rb_funcall(b->io, b->io_partial_read_method, 1, SIZET2NUM(b->io_buffer_size));
         if(b->io_buffer == Qnil) {
             rb_raise(rb_eEOFError, "IO reached end of file");
         }
         StringValue(b->io_buffer);
     } else {
-        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(b->io_buffer_size), b->io_buffer);
+        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(b->io_buffer_size), b->io_buffer);
         if(ret == Qnil) {
             rb_raise(rb_eEOFError, "IO reached end of file");
         }
@@ -646,7 +615,7 @@ size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string,
 {
     if(RSTRING_LEN(string) == 0) {
         /* direct read */
-        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(length), string);
+        VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(length), string);
         if(ret == Qnil) {
             return 0;
         }
@@ -658,7 +627,7 @@ size_t _msgpack_buffer_read_from_io_to_string(msgpack_buffer_t* b, VALUE string,
         b->io_buffer = rb_str_buf_new(0);
     }
 
-    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(length), b->io_buffer);
+    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(length), b->io_buffer);
     if(ret == Qnil) {
         return 0;
     }
@@ -674,7 +643,7 @@ size_t _msgpack_buffer_skip_from_io(msgpack_buffer_t* b, size_t length)
         b->io_buffer = rb_str_buf_new(0);
     }
 
-    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, LONG2NUM(length), b->io_buffer);
+    VALUE ret = rb_funcall(b->io, b->io_partial_read_method, 2, SIZET2NUM(length), b->io_buffer);
     if(ret == Qnil) {
         return 0;
     }

data/ext/msgpack/buffer.h

@@ -49,6 +49,10 @@
 
 #define NO_MAPPED_STRING ((VALUE)0)
 
+#ifndef RB_ENC_INTERNED_STR_NULL_CHECK
+#define RB_ENC_INTERNED_STR_NULL_CHECK 0
+#endif
+
 extern int msgpack_rb_encindex_utf8;
 extern int msgpack_rb_encindex_usascii;
 extern int msgpack_rb_encindex_ascii8bit;
@@ -456,7 +460,11 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
 
 #ifdef HAVE_RB_ENC_INTERNED_STR
     if (will_be_frozen) {
-        result = rb_enc_interned_str(b->read_buffer, length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        if (RB_ENC_INTERNED_STR_NULL_CHECK && length == 0) {
+            result = rb_enc_interned_str("", length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        } else {
+            result = rb_enc_interned_str(b->read_buffer, length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        }
     } else {
         if (utf8) {
             result = rb_utf8_str_new(b->read_buffer, length);

data/ext/msgpack/buffer_class.c

@@ -96,17 +96,17 @@ void MessagePack_Buffer_set_options(msgpack_buffer_t* b, VALUE io, VALUE options
 
         v = rb_hash_aref(options, sym_read_reference_threshold);
         if(v != Qnil) {
-            msgpack_buffer_set_read_reference_threshold(b, NUM2ULONG(v));
+            msgpack_buffer_set_read_reference_threshold(b, NUM2SIZET(v));
         }
 
         v = rb_hash_aref(options, sym_write_reference_threshold);
         if(v != Qnil) {
-            msgpack_buffer_set_write_reference_threshold(b, NUM2ULONG(v));
+            msgpack_buffer_set_write_reference_threshold(b, NUM2SIZET(v));
         }
 
         v = rb_hash_aref(options, sym_io_buffer_size);
         if(v != Qnil) {
-            msgpack_buffer_set_io_buffer_size(b, NUM2ULONG(v));
+            msgpack_buffer_set_io_buffer_size(b, NUM2SIZET(v));
         }
     }
 }
@@ -301,11 +301,11 @@ static VALUE Buffer_skip(VALUE self, VALUE sn)
 
     /* do nothing */
     if(n == 0) {
-        return ULONG2NUM(0);
+        return INT2NUM(0);
     }
 
     size_t sz = read_until_eof(b, Qnil, n);
-    return ULONG2NUM(sz);
+    return SIZET2NUM(sz);
 }
 
 static VALUE Buffer_skip_all(VALUE self, VALUE sn)
@@ -473,7 +473,7 @@ static VALUE Buffer_write_to(VALUE self, VALUE io)
 {
     BUFFER(self, b);
     size_t sz = msgpack_buffer_flush_to_io(b, io, s_write, true);
-    return ULONG2NUM(sz);
+    return SIZET2NUM(sz);
 }
 
 void MessagePack_Buffer_module_init(VALUE mMessagePack)

data/ext/msgpack/compat.h

@@ -22,104 +22,5 @@
 #include "ruby.h"
 #include "ruby/encoding.h"
 
-#if defined(HAVE_RUBY_ST_H)
-#  include "ruby/st.h"  /* ruby hash on Ruby 1.9 */
-#elif defined(HAVE_ST_H)
-#  include "st.h"       /* ruby hash on Ruby 1.8 */
-#endif
-
-
-/*
- * ZALLOC_N (ruby 2.2 or later)
- */
-#ifndef RB_ZALLOC_N
-#  define RB_ZALLOC_N(type,n) ((type*)ruby_xcalloc((size_t)(n),sizeof(type)))
-#endif
-#ifndef ZALLOC_N
-#  define ZALLOC_N(type,n) RB_ZALLOC_N(type,n)
-#endif
-
-
-/*
- * define STR_DUP_LIKELY_DOES_COPY
- * check rb_str_dup actually copies the string or not
- */
-#if defined(RUBY_VM) && defined(FL_ALL) && defined(FL_USER1) && defined(FL_USER3)  /* MRI 1.9 */
-#  define STR_DUP_LIKELY_DOES_COPY(str) FL_ALL(str, FL_USER1|FL_USER3)  /* same as STR_ASSOC_P(str) */
-
-#elif defined(FL_TEST) && defined(ELTS_SHARED)  /* MRI 1.8 */
-#  define STR_DUP_LIKELY_DOES_COPY(str) (!FL_TEST(str, ELTS_SHARED))
-
-//#elif defined(RUBINIUS) || defined(JRUBY)  /* Rubinius and JRuby */
-#else
-#  define STR_DUP_LIKELY_DOES_COPY(str) (1)
-
-#endif
-
-
-/*
- * SIZET2NUM
- */
-#ifndef SIZET2NUM   /* MRI 1.8 */
-#  define SIZET2NUM(v) ULL2NUM(v)
-#endif
-
-
-/*
- * rb_errinfo()
- */
-#if defined(RUBY_VM)  /* MRI 1.9 */
-#  define COMPAT_RERAISE rb_exc_raise(rb_errinfo())
-
-#elif defined(JRUBY)  /* JRuby */
-#  define COMPAT_RERAISE rb_exc_raise(rb_gv_get("$!"))
-
-#else  /* MRI 1.8 and Rubinius */
-#  define COMPAT_RERAISE rb_exc_raise(ruby_errinfo)
-#endif
-
-
-/*
- * RBIGNUM_POSITIVE_P
- */
-#ifndef RBIGNUM_POSITIVE_P
-#  if defined(RUBINIUS)  /* Rubinius <= v1.2.3 */
-#    define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue)
-
-#  elif defined(JRUBY)  /* JRuby */
-#    define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue)
-#    define rb_big2ull(b) rb_num2ull(b)
-     /*#define rb_big2ll(b) rb_num2ll(b)*/
-
-#  else  /* MRI 1.8 */
-#    define RBIGNUM_POSITIVE_P(b) (RBIGNUM(b)->sign)
-#  endif
-#endif
-
-
-/*
- * RSTRING_PTR, RSTRING_LEN
- */
-#ifndef RSTRING_PTR  /* MRI 1.8.5 */
-#  define RSTRING_PTR(s) (RSTRING(s)->ptr)
-#endif
-
-#ifndef RSTRING_LEN  /* MRI 1.8.5 */
-#  define RSTRING_LEN(s) (RSTRING(s)->len)
-#endif
-
-
-/*
- * RSTRUCT_GET
- */
-#ifndef RSTRUCT_GET
-#  ifdef RSTRUCT_PTR  /* MRI <= 2.0.0 */
-#    define RSTRUCT_GET(st, idx)  (RSTRUCT_PTR(st)[idx])
-#  else /* Rubinius */
-#    define RSTRUCT_GET(st, idx)  (rb_struct_aref(st, INT2FIX(idx)))
-#  endif
-#endif
-
-
 #endif
 

data/ext/msgpack/extconf.rb

@@ -2,24 +2,23 @@ require 'mkmf'
 
 have_header("ruby/st.h")
 have_header("st.h")
-have_func("rb_enc_interned_str", "ruby.h")
+have_func("rb_enc_interned_str", "ruby.h") # Ruby 3.0+
+have_func("rb_hash_new_capa", "ruby.h") # Ruby 3.2+
 
 unless RUBY_PLATFORM.include? 'mswin'
-  $CFLAGS << %[ -I.. -Wall -O3 -g -std=gnu99]
+  $CFLAGS << %[ -I.. -Wall -O3 #{RbConfig::CONFIG["debugflags"]} -std=gnu99]
 end
 #$CFLAGS << %[ -DDISABLE_RMEM]
 #$CFLAGS << %[ -DDISABLE_RMEM_REUSE_INTERNAL_FRAGMENT]
 #$CFLAGS << %[ -DDISABLE_BUFFER_READ_REFERENCE_OPTIMIZE]
 #$CFLAGS << %[ -DDISABLE_BUFFER_READ_TO_S_OPTIMIZE]
 
-if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'rbx'
-  # msgpack-ruby doesn't modify data came from RSTRING_PTR(str)
-  $CFLAGS << %[ -DRSTRING_NOT_MODIFIED]
-  # Rubinius C extensions don't grab GVL while rmem is not thread safe
-  $CFLAGS << %[ -DDISABLE_RMEM]
+if RUBY_VERSION.start_with?('3.0.')
+  # https://bugs.ruby-lang.org/issues/18772
+  $CFLAGS << ' -DRB_ENC_INTERNED_STR_NULL_CHECK=1 '
 end
 
-# checking if Hash#[]= (rb_hash_aset) dedupes string keys
+# checking if Hash#[]= (rb_hash_aset) dedupes string keys (Ruby 2.6+)
 h = {}
 x = {}
 r = rand.to_s
@@ -32,7 +31,7 @@ else
 end
 
 
-# checking if String#-@ (str_uminus) dedupes... '
+# checking if String#-@ (str_uminus) dedupes... ' (Ruby 2.5+)
 begin
   a = -(%w(t e s t).join)
   b = -(%w(t e s t).join)
@@ -45,7 +44,7 @@ rescue NoMethodError
   $CFLAGS << ' -DSTR_UMINUS_DEDUPE=0 '
 end
 
-# checking if String#-@ (str_uminus) directly interns frozen strings... '
+# checking if String#-@ (str_uminus) directly interns frozen strings... ' (Ruby 3.0+)
 begin
   s = rand.to_s.freeze
   if (-s).equal?(s) && (-s.dup).equal?(s)
@@ -62,4 +61,3 @@ if warnflags = CONFIG['warnflags']
 end
 
 create_makefile('msgpack/msgpack')
-

data/ext/msgpack/packer.c

@@ -18,24 +18,10 @@
 
 #include "packer.h"
 
-#ifdef RUBINIUS
-static ID s_to_iter;
-static ID s_next;
-static ID s_key;
-static ID s_value;
-#endif
-
 static ID s_call;
 
 void msgpack_packer_static_init()
 {
-#ifdef RUBINIUS
-    s_to_iter = rb_intern("to_iter");
-    s_next = rb_intern("next");
-    s_key = rb_intern("key");
-    s_value = rb_intern("value");
-#endif
-
     s_call = rb_intern("call");
 }
 
@@ -108,17 +94,7 @@ void msgpack_packer_write_hash_value(msgpack_packer_t* pk, VALUE v)
     unsigned int len32 = (unsigned int)len;
     msgpack_packer_write_map_header(pk, len32);
 
-#ifdef RUBINIUS
-    VALUE iter = rb_funcall(v, s_to_iter, 0);
-    VALUE entry = Qnil;
-    while(RTEST(entry = rb_funcall(iter, s_next, 1, entry))) {
-        VALUE key = rb_funcall(entry, s_key, 0);
-        VALUE val = rb_funcall(entry, s_value, 0);
-        write_hash_foreach(key, val, (VALUE) pk);
-    }
-#else
     rb_hash_foreach(v, write_hash_foreach, (VALUE) pk);
-#endif
 }
 
 struct msgpack_call_proc_args_t;

data/ext/msgpack/packer.h

@@ -408,13 +408,7 @@ static inline bool msgpack_packer_is_utf8_compat_string(VALUE v, int encindex)
 {
     return encindex == msgpack_rb_encindex_utf8
         || encindex == msgpack_rb_encindex_usascii
-#ifdef ENC_CODERANGE_ASCIIONLY
-        /* Because ENC_CODERANGE_ASCIIONLY does not scan string, it may return ENC_CODERANGE_UNKNOWN unlike */
-        /* rb_enc_str_asciionly_p. It is always faster than rb_str_encode if it is available. */
-        /* Very old Rubinius (< v1.3.1) doesn't have ENC_CODERANGE_ASCIIONLY. */
-        || (rb_enc_asciicompat(rb_enc_from_index(encindex)) && ENC_CODERANGE_ASCIIONLY(v))
-#endif
-        ;
+        || (rb_enc_asciicompat(rb_enc_from_index(encindex)) && ENC_CODERANGE_ASCIIONLY(v));
 }
 
 static inline void msgpack_packer_write_string_value(msgpack_packer_t* pk, VALUE v)

data/ext/msgpack/packer_class.c

@@ -322,7 +322,7 @@ static VALUE Packer_write_to(VALUE self, VALUE io)
 {
     PACKER(self, pk);
     size_t sz = msgpack_buffer_flush_to_io(PACKER_BUFFER_(pk), io, s_write, true);
-    return ULONG2NUM(sz);
+    return SIZET2NUM(sz);
 }
 
 //static VALUE Packer_append(VALUE self, VALUE string_or_buffer)

data/ext/msgpack/unpacker.c

@@ -34,6 +34,13 @@ static ID s_call;
 static msgpack_rmem_t s_stack_rmem;
 #endif
 
+#if !defined(HAVE_RB_HASH_NEW_CAPA)
+static inline VALUE rb_hash_new_capa(long capa)
+{
+  return rb_hash_new();
+}
+#endif
+
 void msgpack_unpacker_static_init()
 {
 #ifdef UNPACKER_STACK_RMEM
@@ -170,6 +177,9 @@ static inline int object_complete_symbol(msgpack_unpacker_t* uk, VALUE object)
 static inline int object_complete_ext(msgpack_unpacker_t* uk, int ext_type, VALUE str)
 {
     if (uk->optimized_symbol_ext_type && ext_type == uk->symbol_ext_type) {
+        if (RB_UNLIKELY(NIL_P(str))) { // empty extension is returned as Qnil
+            return object_complete_symbol(uk, ID2SYM(rb_intern3("", 0, rb_utf8_encoding())));
+        }
         return object_complete_symbol(uk, rb_str_intern(str));
     }
 
@@ -371,9 +381,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
 
     SWITCH_RANGE(b, 0xa0, 0xbf)  // FixRaw / fixstr
         int count = b & 0x1f;
-        if(count == 0) {
-            return object_complete(uk, rb_utf8_str_new_static("", 0));
-        }
         /* read_raw_body_begin sets uk->reading_raw */
         uk->reading_raw_remaining = count;
         return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -390,7 +397,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         if(count == 0) {
             return object_complete(uk, rb_hash_new());
         }
-        return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new());
+        return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
 
     SWITCH_RANGE(b, 0xc0, 0xdf)  // Variable
         switch(b) {
@@ -473,7 +480,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 uint32_t u32 = _msgpack_be32(cb->u32);
-                return object_complete(uk, ULONG2NUM((unsigned long)u32));
+                return object_complete(uk, ULONG2NUM(u32)); // long at least 32 bits
             }
 
         case 0xcf:  // unsigned int 64
@@ -501,7 +508,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 int32_t i32 = _msgpack_be32(cb->i32);
-                return object_complete(uk, LONG2NUM((long)i32));
+                return object_complete(uk, LONG2NUM(i32)); // long at least 32 bits
             }
 
         case 0xd3:  // signed int 64
@@ -556,9 +563,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
                 uint8_t count = cb->u8;
-                if(count == 0) {
-                    return object_complete(uk, rb_utf8_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -568,9 +572,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
                 uint16_t count = _msgpack_be16(cb->u16);
-                if(count == 0) {
-                    return object_complete(uk, rb_utf8_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -580,9 +581,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 uint32_t count = _msgpack_be32(cb->u32);
-                if(count == 0) {
-                    return object_complete(uk, rb_utf8_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -592,9 +590,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
                 uint8_t count = cb->u8;
-                if(count == 0) {
-                    return object_complete(uk, rb_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -604,9 +599,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
                 uint16_t count = _msgpack_be16(cb->u16);
-                if(count == 0) {
-                    return object_complete(uk, rb_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -616,9 +608,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 uint32_t count = _msgpack_be32(cb->u32);
-                if(count == 0) {
-                    return object_complete(uk, rb_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -651,7 +640,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 if(count == 0) {
                     return object_complete(uk, rb_hash_new());
                 }
-                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new());
+                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
             }
 
         case 0xdf:  // map 32
@@ -661,7 +650,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 if(count == 0) {
                     return object_complete(uk, rb_hash_new());
                 }
-                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new());
+                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
             }
 
         default:

data/ext/msgpack/unpacker_class.c

@@ -209,7 +209,7 @@ static VALUE Unpacker_read_array_header(VALUE self)
         raise_unpacker_error(r);
     }
 
-    return ULONG2NUM(size);
+    return ULONG2NUM(size); // long at least 32 bits
 }
 
 static VALUE Unpacker_read_map_header(VALUE self)
@@ -222,7 +222,7 @@ static VALUE Unpacker_read_map_header(VALUE self)
         raise_unpacker_error((int)r);
     }
 
-    return ULONG2NUM(size);
+    return ULONG2NUM(size); // long at least 32 bits
 }
 
 
@@ -450,4 +450,3 @@ void MessagePack_Unpacker_module_init(VALUE mMessagePack)
 
     rb_define_method(cMessagePack_Unpacker, "full_unpack", Unpacker_full_unpack, 0);
 }
-

data/lib/msgpack/version.rb

@@ -1,5 +1,5 @@
 module MessagePack
-  VERSION = "1.5.2"
+  VERSION = "1.5.4"
   # Note for maintainers:
   #  Don't miss building/releasing the JRuby version (rake buld:java)
   #  See "How to build -java rubygems" in README for more details.

data/msgpack.gemspec

@@ -27,4 +27,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rspec', ['~> 3.3']
   s.add_development_dependency 'yard'
   s.add_development_dependency 'json'
+  s.add_development_dependency 'benchmark-ips', ['~> 2.10.0']
 end

data/spec/factory_spec.rb

@@ -474,6 +474,10 @@ describe MessagePack::Factory do
         expect(roundtrip(:symbol)).to be :symbol
       end
 
+      it 'works with empty symbol' do
+        expect(roundtrip(:"")).to be :""
+      end
+
       it 'preserves encoding for ASCII symbols' do
         expect(:symbol.encoding).to be Encoding::US_ASCII
         expect(roundtrip(:symbol)).to be :symbol

data/spec/spec_helper.rb

@@ -20,7 +20,11 @@ require "msgpack/bigint"
 if GC.respond_to?(:verify_compaction_references)
   # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
   # move objects around, helping to find object movement bugs.
-  GC.verify_compaction_references(double_heap: true, toward: :empty)
+  begin
+    GC.verify_compaction_references(double_heap: true, toward: :empty)
+  rescue NotImplementedError
+    # Some platforms don't support compaction
+  end
 end
 
 if GC.respond_to?(:auto_compact=)

data/spec/unpacker_spec.rb

@@ -707,6 +707,18 @@ describe MessagePack::Unpacker do
         described_class.new(:freeze => true)
       end
 
+      if (-"test").equal?(-"test") # RUBY_VERSION >= "2.5"
+        it 'dedups strings' do
+          interned_str = -"test"
+          roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
+          expect(roundtrip).to be interned_str
+
+          interned_str = -""
+          roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
+          expect(roundtrip).to be interned_str
+        end
+      end
+
       it 'can freeze objects when using .unpack' do
         parsed_struct = MessagePack.unpack(buffer, freeze: true)
         parsed_struct.should == struct

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: msgpack
 version: !ruby/object:Gem::Version
-  version: 1.5.2
+  version: 1.5.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-27 00:00:00.000000000 Z
+date: 2022-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -96,6 +96,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.0
 description: MessagePack is a binary-based efficient object serialization library.
   It enables to exchange structured objects between many languages like JSON. But
   unlike JSON, it is very fast and small.
@@ -117,16 +131,7 @@ files:
 - README.md
 - Rakefile
 - appveyor.yml
-- bench/pack.rb
-- bench/pack_log.rb
-- bench/pack_log_long.rb
-- bench/pack_symbols.rb
-- bench/run.sh
-- bench/run_long.sh
-- bench/run_symbols.sh
-- bench/unpack.rb
-- bench/unpack_log.rb
-- bench/unpack_log_long.rb
+- bench/bench.rb
 - doclib/msgpack.rb
 - doclib/msgpack/buffer.rb
 - doclib/msgpack/core_ext.rb
@@ -231,7 +236,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: MessagePack, a binary-based efficient data interchange format.

data/bench/pack.rb

@@ -1,23 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data = { 'hello' => 'world', 'nested' => ['structure', {value: 42}] }
-data_sym = { hello: 'world', nested: ['structure', {value: 42}] }
-
-data = MessagePack.pack(:hello => 'world', :nested => ['structure', {:value => 42}])
-
-Viiite.bench do |b|
-  b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
-    b.report(:strings) do
-      runs.times do
-        MessagePack.pack(data)
-      end
-    end
-
-    b.report(:symbols) do
-      runs.times do
-        MessagePack.pack(data_sym)
-      end
-    end
-  end
-end

data/bench/pack_log.rb

@@ -1,33 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data_plain = { 'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"' }
-data_structure = {
-  'remote_host' => '127.0.0.1',
-  'remote_user' => '-',
-  'date' => '10/Oct/2000:13:55:36 -0700',
-  'request' => 'GET /apache_pb.gif HTTP/1.0',
-  'method' => 'GET',
-  'path' => '/apache_pb.gif',
-  'protocol' => 'HTTP/1.0',
-  'status' => 200,
-  'bytes' => 2326,
-  'referer' => 'http://www.example.com/start.html',
-  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
-}
-
-Viiite.bench do |b|
-  b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
-    b.report(:plain) do
-      runs.times do
-        MessagePack.pack(data_plain)
-      end
-    end
-
-    b.report(:structure) do
-      runs.times do
-        MessagePack.pack(data_structure)
-      end
-    end
-  end
-end

data/bench/pack_log_long.rb

@@ -1,65 +0,0 @@
-# viiite report --regroup bench,threads bench/pack_log_long.rb
-
-require 'viiite'
-require 'msgpack'
-
-data_plain = { 'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"' }
-data_structure = {
-  'remote_host' => '127.0.0.1',
-  'remote_user' => '-',
-  'date' => '10/Oct/2000:13:55:36 -0700',
-  'request' => 'GET /apache_pb.gif HTTP/1.0',
-  'method' => 'GET',
-  'path' => '/apache_pb.gif',
-  'protocol' => 'HTTP/1.0',
-  'status' => 200,
-  'bytes' => 2326,
-  'referer' => 'http://www.example.com/start.html',
-  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
-}
-
-seconds = 3600 # 1 hour
-
-Viiite.bench do |b|
-  b.range_over([1, 2, 4, 8, 16], :threads) do |threads|
-    b.report(:plain) do
-      ths = []
-      end_at = Time.now + seconds
-      threads.times do
-        t = Thread.new do
-          packs = 0
-          while Time.now < end_at
-            10000.times do
-              MessagePack.pack(data_plain)
-            end
-            packs += 10000
-          end
-          packs
-        end
-        ths.push t
-      end
-      sum = ths.reduce(0){|r,t| r + t.value }
-      puts "MessagePack.pack, plain, #{threads} threads: #{sum} times, #{sum / seconds} times/second."
-    end
-
-    b.report(:structure) do
-      ths = []
-      end_at = Time.now + seconds
-      threads.times do
-        t = Thread.new do
-          packs = 0
-          while Time.now < end_at
-            10000.times do
-              MessagePack.pack(data_structure)
-            end
-            packs += 10000
-          end
-          packs
-        end
-        ths.push t
-      end
-      sum = ths.reduce(0){|r,t| r + t.value }
-      puts "MessagePack.pack, structured, #{threads} threads: #{sum} times, #{sum / seconds} times/second."
-    end
-  end
-end

data/bench/pack_symbols.rb

@@ -1,28 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data = :symbol
-
-Viiite.bench do |b|
-  b.variation_point :branch, `git rev-parse --abbrev-ref HEAD`
-
-  b.range_over([:symbol, :none], :reg_type) do |reg_type|
-    packer = MessagePack::Packer.new
-    packer.register_type(0x00, Symbol, :to_msgpack_ext) if reg_type == :symbol
-
-    b.range_over([100_000, 1_000_000, 10_000_000], :count) do |count|
-      packer.clear
-      b.report(:multi_run) do
-        count.times do
-          packer.pack(data)
-        end
-      end
-
-      packer.clear
-      items_data = [].fill(data, 0, count)
-      b.report(:large_run) do
-        packer.pack(items_data)
-      end
-    end
-  end
-end

data/bench/run.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# prerequisites
-# $ rbenv shell 2.2.1 (or jruby-x.x.x or ...)
-# $ rake install
-
-echo "pack"
-viiite report --regroup bench,runs bench/pack.rb 
-echo "unpack"
-viiite report --regroup bench,runs bench/unpack.rb 
-echo "pack log"
-viiite report --regroup bench,runs bench/pack_log.rb 
-echo "unpack log"
-viiite report --regroup bench,runs bench/unpack_log.rb 

data/bench/run_long.sh

@@ -1,35 +0,0 @@
-#!/bin/sh
-
-# prerequisites
-# $ sudo apt-get install sysstat
-# $ rbenv shell 2.2.1 (or jruby-x.x.x or ...)
-# $ rake install
-
-# 60 * 600 : 60*60 * 5[threads] * 2[bench]
-
-ruby -v
-
-echo "pack log long"
-viiite report --regroup bench,threads bench/pack_log_long.rb &
-sar -o pack_log_long.sar -r 60 600 > /dev/null 2>&1 &
-
-declare -i i=0
-while [ $i -lt 600 ]; do
-    ps auxww | grep ruby | grep -v grep | awk '{print $5,$6;}' >> pack_log_long.mem.txt
-    i=i+1
-    sleep 60
-done
-
-sleep 120 # cool down
-
-echo "unpack log long"
-viiite report --regroup bench,threads bench/unpack_log_long.rb &
-sar -o unpack_log_long.sar -r 60 600 > /dev/null 2>&1 &
-
-i=0
-while [ $i -lt 600 ]; do
-    ps auxww | grep ruby | grep -v grep | awk '{print $5,$6;}' >> pack_log_long.mem.txt
-    i=i+1
-    sleep 60
-done
-

data/bench/run_symbols.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-# so master and this branch have the benchmark file in any case
-cp bench/pack_symbols.rb bench/pack_symbols_tmp.rb
-
-benchmark=""
-current_branch=`git rev-parse --abbrev-ref HEAD`
-
-for branch in master $current_branch; do
-    echo "Testing branch $branch"
-    git checkout $branch
-
-    echo "Installing gem..."
-    rake install
-
-    echo "Running benchmark..."
-    if [ "$benchmark" ]; then
-        benchmark+=$'\n'
-    fi
-    benchmark+=$(viiite run bench/pack_symbols_tmp.rb)
-    echo
-done
-
-rm bench/pack_symbols_tmp.rb
-
-echo "$benchmark" | viiite report --regroup bench,reg_type,count,branch

data/bench/unpack.rb

@@ -1,21 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data = MessagePack.pack(:hello => 'world', :nested => ['structure', {:value => 42}])
-
-Viiite.bench do |b|
-  b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
-    b.report(:strings) do
-      runs.times do
-        MessagePack.unpack(data)
-      end
-    end
-
-    b.report(:symbols) do
-      options = {:symbolize_keys => true}
-      runs.times do
-        MessagePack.unpack(data, options)
-      end
-    end
-  end
-end

data/bench/unpack_log.rb

@@ -1,34 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data_plain = MessagePack.pack({ 'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"' })
-
-data_structure = MessagePack.pack({
-  'remote_host' => '127.0.0.1',
-  'remote_user' => '-',
-  'date' => '10/Oct/2000:13:55:36 -0700',
-  'request' => 'GET /apache_pb.gif HTTP/1.0',
-  'method' => 'GET',
-  'path' => '/apache_pb.gif',
-  'protocol' => 'HTTP/1.0',
-  'status' => 200,
-  'bytes' => 2326,
-  'referer' => 'http://www.example.com/start.html',
-  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
-})
-
-Viiite.bench do |b|
-  b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
-    b.report(:plain) do
-      runs.times do
-        MessagePack.unpack(data_plain)
-      end
-    end
-
-    b.report(:structure) do
-      runs.times do
-        MessagePack.unpack(data_structure)
-      end
-    end
-  end
-end

data/bench/unpack_log_long.rb

@@ -1,67 +0,0 @@
-# viiite report --regroup bench,threads bench/pack_log_long.rb
-
-require 'viiite'
-require 'msgpack'
-
-data_plain = MessagePack.pack({
-    'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"'
-})
-data_structure = MessagePack.pack({
-  'remote_host' => '127.0.0.1',
-  'remote_user' => '-',
-  'date' => '10/Oct/2000:13:55:36 -0700',
-  'request' => 'GET /apache_pb.gif HTTP/1.0',
-  'method' => 'GET',
-  'path' => '/apache_pb.gif',
-  'protocol' => 'HTTP/1.0',
-  'status' => 200,
-  'bytes' => 2326,
-  'referer' => 'http://www.example.com/start.html',
-  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
-})
-
-seconds = 3600 # 1 hour
-
-Viiite.bench do |b|
-  b.range_over([1, 2, 4, 8, 16], :threads) do |threads|
-    b.report(:plain) do
-      ths = []
-      end_at = Time.now + seconds
-      threads.times do
-        t = Thread.new do
-          packs = 0
-          while Time.now < end_at
-            10000.times do
-              MessagePack.unpack(data_plain)
-            end
-            packs += 10000
-          end
-          packs
-        end
-        ths.push t
-      end
-      sum = ths.reduce(0){|r,t| r + t.value }
-      puts "MessagePack.unpack, plain, #{threads} threads: #{sum} times, #{sum / seconds} times/second."
-    end
-
-    b.report(:structure) do
-      ths = []
-      end_at = Time.now + seconds
-      threads.times do
-        t = Thread.new do
-          packs = 0
-          while Time.now < end_at
-            10000.times do
-              MessagePack.unpack(data_structure)
-            end
-            packs += 10000
-          end
-          packs
-        end
-        ths.push t
-      end
-      sum = ths.reduce(0){|r,t| r + t.value }
-      puts "MessagePack.unpack, structured, #{threads} threads: #{sum} times, #{sum / seconds} times/second."
-    end
-  end
-end