msgpack 1.5.2 → 1.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7de0658b1f87162270c3cc2aac70ffff8d408d974faac6ff5ad009fa11dba4dd
-  data.tar.gz: 9da35f2cc47ced45efd96b10ad8a3b491f0658b5ba247ad7b4aa22b901c66110
+  metadata.gz: 6c335ddbadfa205f9fabd011bdcbe768924a532e762550a8c974035eb2b6ca59
+  data.tar.gz: 24abdf9bb3b34484ded41acc0ac4291ba25ee48660cca66968b265453b35ed96
 SHA512:
-  metadata.gz: b23c683b4c7fbbedd8dedc5d6eb8f5d6778aa3d3f7afef839cfe3227fc2edff713d9f17d6c8ce3808a6103325d7923ca6abfaf8792dba11485ed9d067fe6aa42
-  data.tar.gz: ced82946481b00d99a69aea792efbeb93982d39836bc763c7c4a0f71844ca48044cff1f58c9695aa4ebb19d0be6a0b43f2e3b4af04ca06d7ad4bfc798cacd0b8
+  metadata.gz: d0ebbe1cca5508d5a1fe4b28d7558e1cbe35e82bdf3b4596e3998feb86298e4bba718153a275ab808bf83651225f20cea52e1117df816e629cc05fbf2513f53c
+  data.tar.gz: 17ed1e47c07f1388a78b844ff2621414531eea6287b020a1568c9f2b6c2d3e8cbacff6e18a00159604fc241676cdd7a1542af4c4a41f1fc0795945912e885d6f

data/ChangeLog

@@ -1,3 +1,8 @@
+2022-05-30 version 1.5.3:
+
+* Fix deduplication of empty strings when using the `freeze: true` option.
+* Use `rb_hash_new_capa` when available (Ruby 3.2) for improved performance when parsing large hashes.
+
 2022-05-27 version 1.5.2:
 
 * Fix bug about unpacking ext type objects with the recursive option

data/ext/msgpack/buffer.c

@@ -300,30 +300,6 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
     }
 }
 
-static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE string)
-{
-    VALUE mapped_string = rb_str_dup(string);
-    ENCODING_SET(mapped_string, msgpack_rb_encindex_ascii8bit);
-
-    _msgpack_buffer_add_new_chunk(b);
-
-    char* data = RSTRING_PTR(mapped_string);
-    size_t length = RSTRING_LEN(mapped_string);
-
-    b->tail.first = (char*) data;
-    b->tail.last = (char*) data + length;
-    b->tail.mapped_string = mapped_string;
-    b->tail.mem = NULL;
-
-    /* msgpack_buffer_writable_size should return 0 for mapped chunk */
-    b->tail_buffer_end = b->tail.last;
-
-    /* consider read_buffer */
-    if(b->head == &b->tail) {
-        b->read_buffer = b->tail.first;
-    }
-}
-
 void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
 {
     size_t length = RSTRING_LEN(string);
@@ -332,16 +308,9 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
         msgpack_buffer_flush(b);
         if (ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit) {
             rb_funcall(b->io, b->io_write_all_method, 1, string);
-        } else if(!STR_DUP_LIKELY_DOES_COPY(string)) {
-            VALUE s = rb_str_dup(string);
-            ENCODING_SET(s, msgpack_rb_encindex_ascii8bit);
-            rb_funcall(b->io, b->io_write_all_method, 1, s);
         } else {
             msgpack_buffer_append(b, RSTRING_PTR(string), length);
         }
-    } else if(!STR_DUP_LIKELY_DOES_COPY(string)) {
-        _msgpack_buffer_append_reference(b, string);
-
     } else {
         msgpack_buffer_append(b, RSTRING_PTR(string), length);
     }

data/ext/msgpack/buffer.h

@@ -49,6 +49,10 @@
 
 #define NO_MAPPED_STRING ((VALUE)0)
 
+#ifndef RB_ENC_INTERNED_STR_NULL_CHECK
+#define RB_ENC_INTERNED_STR_NULL_CHECK 0
+#endif
+
 extern int msgpack_rb_encindex_utf8;
 extern int msgpack_rb_encindex_usascii;
 extern int msgpack_rb_encindex_ascii8bit;
@@ -456,7 +460,11 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
 
 #ifdef HAVE_RB_ENC_INTERNED_STR
     if (will_be_frozen) {
-        result = rb_enc_interned_str(b->read_buffer, length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        if (RB_ENC_INTERNED_STR_NULL_CHECK && length == 0) {
+            result = rb_enc_interned_str("", length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        } else {
+            result = rb_enc_interned_str(b->read_buffer, length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+        }
     } else {
         if (utf8) {
             result = rb_utf8_str_new(b->read_buffer, length);

data/ext/msgpack/compat.h

@@ -22,104 +22,5 @@
 #include "ruby.h"
 #include "ruby/encoding.h"
 
-#if defined(HAVE_RUBY_ST_H)
-#  include "ruby/st.h"  /* ruby hash on Ruby 1.9 */
-#elif defined(HAVE_ST_H)
-#  include "st.h"       /* ruby hash on Ruby 1.8 */
-#endif
-
-
-/*
- * ZALLOC_N (ruby 2.2 or later)
- */
-#ifndef RB_ZALLOC_N
-#  define RB_ZALLOC_N(type,n) ((type*)ruby_xcalloc((size_t)(n),sizeof(type)))
-#endif
-#ifndef ZALLOC_N
-#  define ZALLOC_N(type,n) RB_ZALLOC_N(type,n)
-#endif
-
-
-/*
- * define STR_DUP_LIKELY_DOES_COPY
- * check rb_str_dup actually copies the string or not
- */
-#if defined(RUBY_VM) && defined(FL_ALL) && defined(FL_USER1) && defined(FL_USER3)  /* MRI 1.9 */
-#  define STR_DUP_LIKELY_DOES_COPY(str) FL_ALL(str, FL_USER1|FL_USER3)  /* same as STR_ASSOC_P(str) */
-
-#elif defined(FL_TEST) && defined(ELTS_SHARED)  /* MRI 1.8 */
-#  define STR_DUP_LIKELY_DOES_COPY(str) (!FL_TEST(str, ELTS_SHARED))
-
-//#elif defined(RUBINIUS) || defined(JRUBY)  /* Rubinius and JRuby */
-#else
-#  define STR_DUP_LIKELY_DOES_COPY(str) (1)
-
-#endif
-
-
-/*
- * SIZET2NUM
- */
-#ifndef SIZET2NUM   /* MRI 1.8 */
-#  define SIZET2NUM(v) ULL2NUM(v)
-#endif
-
-
-/*
- * rb_errinfo()
- */
-#if defined(RUBY_VM)  /* MRI 1.9 */
-#  define COMPAT_RERAISE rb_exc_raise(rb_errinfo())
-
-#elif defined(JRUBY)  /* JRuby */
-#  define COMPAT_RERAISE rb_exc_raise(rb_gv_get("$!"))
-
-#else  /* MRI 1.8 and Rubinius */
-#  define COMPAT_RERAISE rb_exc_raise(ruby_errinfo)
-#endif
-
-
-/*
- * RBIGNUM_POSITIVE_P
- */
-#ifndef RBIGNUM_POSITIVE_P
-#  if defined(RUBINIUS)  /* Rubinius <= v1.2.3 */
-#    define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue)
-
-#  elif defined(JRUBY)  /* JRuby */
-#    define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue)
-#    define rb_big2ull(b) rb_num2ull(b)
-     /*#define rb_big2ll(b) rb_num2ll(b)*/
-
-#  else  /* MRI 1.8 */
-#    define RBIGNUM_POSITIVE_P(b) (RBIGNUM(b)->sign)
-#  endif
-#endif
-
-
-/*
- * RSTRING_PTR, RSTRING_LEN
- */
-#ifndef RSTRING_PTR  /* MRI 1.8.5 */
-#  define RSTRING_PTR(s) (RSTRING(s)->ptr)
-#endif
-
-#ifndef RSTRING_LEN  /* MRI 1.8.5 */
-#  define RSTRING_LEN(s) (RSTRING(s)->len)
-#endif
-
-
-/*
- * RSTRUCT_GET
- */
-#ifndef RSTRUCT_GET
-#  ifdef RSTRUCT_PTR  /* MRI <= 2.0.0 */
-#    define RSTRUCT_GET(st, idx)  (RSTRUCT_PTR(st)[idx])
-#  else /* Rubinius */
-#    define RSTRUCT_GET(st, idx)  (rb_struct_aref(st, INT2FIX(idx)))
-#  endif
-#endif
-
-
 #endif
 

data/ext/msgpack/extconf.rb

@@ -2,7 +2,8 @@ require 'mkmf'
 
 have_header("ruby/st.h")
 have_header("st.h")
-have_func("rb_enc_interned_str", "ruby.h")
+have_func("rb_enc_interned_str", "ruby.h") # Ruby 3.0+
+have_func("rb_hash_new_capa", "ruby.h") # Ruby 3.2+
 
 unless RUBY_PLATFORM.include? 'mswin'
   $CFLAGS << %[ -I.. -Wall -O3 -g -std=gnu99]
@@ -12,14 +13,12 @@ end
 #$CFLAGS << %[ -DDISABLE_BUFFER_READ_REFERENCE_OPTIMIZE]
 #$CFLAGS << %[ -DDISABLE_BUFFER_READ_TO_S_OPTIMIZE]
 
-if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'rbx'
-  # msgpack-ruby doesn't modify data came from RSTRING_PTR(str)
-  $CFLAGS << %[ -DRSTRING_NOT_MODIFIED]
-  # Rubinius C extensions don't grab GVL while rmem is not thread safe
-  $CFLAGS << %[ -DDISABLE_RMEM]
+if RUBY_VERSION.start_with?('3.0.')
+  # https://bugs.ruby-lang.org/issues/18772
+  $CFLAGS << ' -DRB_ENC_INTERNED_STR_NULL_CHECK=1 '
 end
 
-# checking if Hash#[]= (rb_hash_aset) dedupes string keys
+# checking if Hash#[]= (rb_hash_aset) dedupes string keys (Ruby 2.6+)
 h = {}
 x = {}
 r = rand.to_s
@@ -32,7 +31,7 @@ else
 end
 
 
-# checking if String#-@ (str_uminus) dedupes... '
+# checking if String#-@ (str_uminus) dedupes... ' (Ruby 2.5+)
 begin
   a = -(%w(t e s t).join)
   b = -(%w(t e s t).join)
@@ -45,7 +44,7 @@ rescue NoMethodError
   $CFLAGS << ' -DSTR_UMINUS_DEDUPE=0 '
 end
 
-# checking if String#-@ (str_uminus) directly interns frozen strings... '
+# checking if String#-@ (str_uminus) directly interns frozen strings... ' (Ruby 3.0+)
 begin
   s = rand.to_s.freeze
   if (-s).equal?(s) && (-s.dup).equal?(s)
@@ -62,4 +61,3 @@ if warnflags = CONFIG['warnflags']
 end
 
 create_makefile('msgpack/msgpack')
-

data/ext/msgpack/packer.c

@@ -18,24 +18,10 @@
 
 #include "packer.h"
 
-#ifdef RUBINIUS
-static ID s_to_iter;
-static ID s_next;
-static ID s_key;
-static ID s_value;
-#endif
-
 static ID s_call;
 
 void msgpack_packer_static_init()
 {
-#ifdef RUBINIUS
-    s_to_iter = rb_intern("to_iter");
-    s_next = rb_intern("next");
-    s_key = rb_intern("key");
-    s_value = rb_intern("value");
-#endif
-
     s_call = rb_intern("call");
 }
 
@@ -108,17 +94,7 @@ void msgpack_packer_write_hash_value(msgpack_packer_t* pk, VALUE v)
     unsigned int len32 = (unsigned int)len;
     msgpack_packer_write_map_header(pk, len32);
 
-#ifdef RUBINIUS
-    VALUE iter = rb_funcall(v, s_to_iter, 0);
-    VALUE entry = Qnil;
-    while(RTEST(entry = rb_funcall(iter, s_next, 1, entry))) {
-        VALUE key = rb_funcall(entry, s_key, 0);
-        VALUE val = rb_funcall(entry, s_value, 0);
-        write_hash_foreach(key, val, (VALUE) pk);
-    }
-#else
     rb_hash_foreach(v, write_hash_foreach, (VALUE) pk);
-#endif
 }
 
 struct msgpack_call_proc_args_t;

data/ext/msgpack/packer.h

@@ -408,13 +408,7 @@ static inline bool msgpack_packer_is_utf8_compat_string(VALUE v, int encindex)
 {
     return encindex == msgpack_rb_encindex_utf8
         || encindex == msgpack_rb_encindex_usascii
-#ifdef ENC_CODERANGE_ASCIIONLY
-        /* Because ENC_CODERANGE_ASCIIONLY does not scan string, it may return ENC_CODERANGE_UNKNOWN unlike */
-        /* rb_enc_str_asciionly_p. It is always faster than rb_str_encode if it is available. */
-        /* Very old Rubinius (< v1.3.1) doesn't have ENC_CODERANGE_ASCIIONLY. */
-        || (rb_enc_asciicompat(rb_enc_from_index(encindex)) && ENC_CODERANGE_ASCIIONLY(v))
-#endif
-        ;
+        || (rb_enc_asciicompat(rb_enc_from_index(encindex)) && ENC_CODERANGE_ASCIIONLY(v));
 }
 
 static inline void msgpack_packer_write_string_value(msgpack_packer_t* pk, VALUE v)

data/ext/msgpack/unpacker.c

@@ -34,6 +34,13 @@ static ID s_call;
 static msgpack_rmem_t s_stack_rmem;
 #endif
 
+#if !defined(HAVE_RB_HASH_NEW_CAPA)
+static inline VALUE rb_hash_new_capa(long capa)
+{
+  return rb_hash_new();
+}
+#endif
+
 void msgpack_unpacker_static_init()
 {
 #ifdef UNPACKER_STACK_RMEM
@@ -371,9 +378,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
 
     SWITCH_RANGE(b, 0xa0, 0xbf)  // FixRaw / fixstr
         int count = b & 0x1f;
-        if(count == 0) {
-            return object_complete(uk, rb_utf8_str_new_static("", 0));
-        }
         /* read_raw_body_begin sets uk->reading_raw */
         uk->reading_raw_remaining = count;
         return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -390,7 +394,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
         if(count == 0) {
             return object_complete(uk, rb_hash_new());
         }
-        return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new());
+        return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
 
     SWITCH_RANGE(b, 0xc0, 0xdf)  // Variable
         switch(b) {
@@ -556,9 +560,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
                 uint8_t count = cb->u8;
-                if(count == 0) {
-                    return object_complete(uk, rb_utf8_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -568,9 +569,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
                 uint16_t count = _msgpack_be16(cb->u16);
-                if(count == 0) {
-                    return object_complete(uk, rb_utf8_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -580,9 +578,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 uint32_t count = _msgpack_be32(cb->u32);
-                if(count == 0) {
-                    return object_complete(uk, rb_utf8_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_STRING);
@@ -592,9 +587,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
                 uint8_t count = cb->u8;
-                if(count == 0) {
-                    return object_complete(uk, rb_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -604,9 +596,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
                 uint16_t count = _msgpack_be16(cb->u16);
-                if(count == 0) {
-                    return object_complete(uk, rb_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -616,9 +605,6 @@ static int read_primitive(msgpack_unpacker_t* uk)
             {
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 uint32_t count = _msgpack_be32(cb->u32);
-                if(count == 0) {
-                    return object_complete(uk, rb_str_new_static("", 0));
-                }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
                 return read_raw_body_begin(uk, RAW_TYPE_BINARY);
@@ -651,7 +637,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 if(count == 0) {
                     return object_complete(uk, rb_hash_new());
                 }
-                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new());
+                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
             }
 
         case 0xdf:  // map 32
@@ -661,7 +647,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 if(count == 0) {
                     return object_complete(uk, rb_hash_new());
                 }
-                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new());
+                return _msgpack_unpacker_stack_push(uk, STACK_TYPE_MAP_KEY, count*2, rb_hash_new_capa(count));
             }
 
         default:

data/ext/msgpack/unpacker_class.c

@@ -450,4 +450,3 @@ void MessagePack_Unpacker_module_init(VALUE mMessagePack)
 
     rb_define_method(cMessagePack_Unpacker, "full_unpack", Unpacker_full_unpack, 0);
 }
-

data/lib/msgpack/version.rb

@@ -1,5 +1,5 @@
 module MessagePack
-  VERSION = "1.5.2"
+  VERSION = "1.5.3"
   # Note for maintainers:
   #  Don't miss building/releasing the JRuby version (rake buld:java)
   #  See "How to build -java rubygems" in README for more details.

data/spec/spec_helper.rb

@@ -20,7 +20,11 @@ require "msgpack/bigint"
 if GC.respond_to?(:verify_compaction_references)
   # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
   # move objects around, helping to find object movement bugs.
-  GC.verify_compaction_references(double_heap: true, toward: :empty)
+  begin
+    GC.verify_compaction_references(double_heap: true, toward: :empty)
+  rescue NotImplementedError
+    # Some platforms don't support compaction
+  end
 end
 
 if GC.respond_to?(:auto_compact=)

data/spec/unpacker_spec.rb

@@ -707,6 +707,18 @@ describe MessagePack::Unpacker do
         described_class.new(:freeze => true)
       end
 
+      if (-"test").equal?(-"test") # RUBY_VERSION >= "2.5"
+        it 'dedups strings' do
+          interned_str = -"test"
+          roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
+          expect(roundtrip).to be interned_str
+
+          interned_str = -""
+          roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
+          expect(roundtrip).to be interned_str
+        end
+      end
+
       it 'can freeze objects when using .unpack' do
         parsed_struct = MessagePack.unpack(buffer, freeze: true)
         parsed_struct.should == struct

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: msgpack
 version: !ruby/object:Gem::Version
-  version: 1.5.2
+  version: 1.5.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-27 00:00:00.000000000 Z
+date: 2022-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -231,7 +231,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: MessagePack, a binary-based efficient data interchange format.