msgpack 1.4.0.pre1 → 1.4.2

data/ext/msgpack/buffer.h

@@ -49,11 +49,11 @@
 
 #define NO_MAPPED_STRING ((VALUE)0)
 
-#ifdef COMPAT_HAVE_ENCODING  /* see compat.h*/
 extern int msgpack_rb_encindex_utf8;
 extern int msgpack_rb_encindex_usascii;
 extern int msgpack_rb_encindex_ascii8bit;
-#endif
+
+extern ID s_uminus;
 
 struct msgpack_buffer_chunk_t;
 typedef struct msgpack_buffer_chunk_t msgpack_buffer_chunk_t;
@@ -438,7 +438,7 @@ static inline VALUE _msgpack_buffer_refer_head_mapped_string(msgpack_buffer_t* b
     return rb_str_substr(b->head->mapped_string, offset, length);
 }
 
-static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_t length, bool will_be_frozen)
+static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_t length, bool will_be_frozen, bool utf8)
 {
 #ifndef DISABLE_BUFFER_READ_REFERENCE_OPTIMIZE
     /* optimize */
@@ -446,16 +446,52 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
             b->head->mapped_string != NO_MAPPED_STRING &&
             length >= b->read_reference_threshold) {
         VALUE result = _msgpack_buffer_refer_head_mapped_string(b, length);
+        if (utf8) ENCODING_SET(result, msgpack_rb_encindex_utf8);
         _msgpack_buffer_consumed(b, length);
         return result;
     }
 #endif
 
-    VALUE result = rb_str_new(b->read_buffer, length);
+    VALUE result;
+
+#ifdef HAVE_RB_ENC_INTERNED_STR
+    if (will_be_frozen) {
+        result = rb_enc_interned_str(b->read_buffer, length, utf8 ? rb_utf8_encoding() : rb_ascii8bit_encoding());
+    } else {
+        if (utf8) {
+            result = rb_utf8_str_new(b->read_buffer, length);
+        } else {
+            result = rb_str_new(b->read_buffer, length);
+        }
+    }
     _msgpack_buffer_consumed(b, length);
     return result;
-}
 
+#else
 
-#endif
+    if (utf8) {
+        result = rb_utf8_str_new(b->read_buffer, length);
+    } else {
+        result = rb_str_new(b->read_buffer, length);
+    }
+
+#if STR_UMINUS_DEDUPE
+    if (will_be_frozen) {
+#if STR_UMINUS_DEDUPE_FROZEN
+        // Starting from MRI 2.8 it is preferable to freeze the string
+        // before deduplication so that it can be interned directly
+        // otherwise it would be duplicated first which is wasteful.
+        rb_str_freeze(result);
+#endif //STR_UMINUS_DEDUPE_FROZEN
+        // MRI 2.5 and older do not deduplicate strings that are already
+        // frozen.
+        result = rb_funcall(result, s_uminus, 0);
+    }
+#endif // STR_UMINUS_DEDUPE
+    _msgpack_buffer_consumed(b, length);
+    return result;
+
+#endif // HAVE_RB_ENC_INTERNED_STR
+}
 
+#endif

data/ext/msgpack/compat.h

@@ -20,6 +20,7 @@
 
 #include <stdbool.h>
 #include "ruby.h"
+#include "ruby/encoding.h"
 
 #if defined(HAVE_RUBY_ST_H)
 #  include "ruby/st.h"  /* ruby hash on Ruby 1.9 */
@@ -38,18 +39,6 @@
 #  define ZALLOC_N(type,n) RB_ZALLOC_N(type,n)
 #endif
 
-/*
- * COMPAT_HAVE_ENCODING
- */
-#ifdef HAVE_RUBY_ENCODING_H
-#  include "ruby/encoding.h"
-#  define COMPAT_HAVE_ENCODING
-#endif
-
-#if defined(__MACRUBY__)  /* MacRuby */
-#  undef COMPAT_HAVE_ENCODING
-#endif
-
 
 /*
  * define STR_DUP_LIKELY_DOES_COPY

data/ext/msgpack/extconf.rb

@@ -4,6 +4,7 @@ have_header("ruby/st.h")
 have_header("st.h")
 have_func("rb_str_replace", ["ruby.h"])
 have_func("rb_intern_str", ["ruby.h"])
+have_func("rb_enc_interned_str", "ruby.h")
 have_func("rb_sym2str", ["ruby.h"])
 have_func("rb_str_intern", ["ruby.h"])
 have_func("rb_block_lambda", ["ruby.h"])
@@ -37,6 +38,32 @@ else
   $CFLAGS << ' -DHASH_ASET_DEDUPE=0 '
 end
 
+
+# checking if String#-@ (str_uminus) dedupes... '
+begin
+  a = -(%w(t e s t).join)
+  b = -(%w(t e s t).join)
+  if a.equal?(b)
+    $CFLAGS << ' -DSTR_UMINUS_DEDUPE=1 '
+  else
+    $CFLAGS += ' -DSTR_UMINUS_DEDUPE=0 '
+  end
+rescue NoMethodError
+  $CFLAGS << ' -DSTR_UMINUS_DEDUPE=0 '
+end
+
+# checking if String#-@ (str_uminus) directly interns frozen strings... '
+begin
+  s = rand.to_s.freeze
+  if (-s).equal?(s) && (-s.dup).equal?(s)
+    $CFLAGS << ' -DSTR_UMINUS_DEDUPE_FROZEN=1 '
+  else
+    $CFLAGS << ' -DSTR_UMINUS_DEDUPE_FROZEN=0 '
+  end
+rescue NoMethodError
+  $CFLAGS << ' -DSTR_UMINUS_DEDUPE_FROZEN=0 '
+end
+
 if warnflags = CONFIG['warnflags']
   warnflags.slice!(/ -Wdeclaration-after-statement/)
 end

data/ext/msgpack/packer.h

@@ -396,7 +396,6 @@ static inline void msgpack_packer_write_ext(msgpack_packer_t* pk, int ext_type,
     msgpack_buffer_append_string(PACKER_BUFFER_(pk), payload);
 }
 
-#ifdef COMPAT_HAVE_ENCODING
 static inline bool msgpack_packer_is_binary(VALUE v, int encindex)
 {
     return encindex == msgpack_rb_encindex_ascii8bit;
@@ -414,7 +413,6 @@ static inline bool msgpack_packer_is_utf8_compat_string(VALUE v, int encindex)
 #endif
         ;
 }
-#endif
 
 static inline void msgpack_packer_write_string_value(msgpack_packer_t* pk, VALUE v)
 {
@@ -425,7 +423,6 @@ static inline void msgpack_packer_write_string_value(msgpack_packer_t* pk, VALUE
         rb_raise(rb_eArgError, "size of string is too long to pack: %lu bytes should be <= %lu", len, 0xffffffffUL);
     }
 
-#ifdef COMPAT_HAVE_ENCODING
     int encindex = ENCODING_GET(v);
     if(msgpack_packer_is_binary(v, encindex) && !pk->compatibility_mode) {
         /* write ASCII-8BIT string using Binary type */
@@ -443,10 +440,6 @@ static inline void msgpack_packer_write_string_value(msgpack_packer_t* pk, VALUE
         msgpack_packer_write_raw_header(pk, (unsigned int)len);
         msgpack_buffer_append_string(PACKER_BUFFER_(pk), v);
     }
-#else
-    msgpack_packer_write_raw_header(pk, (unsigned int)len);
-    msgpack_buffer_append_string(PACKER_BUFFER_(pk), v);
-#endif
 }
 
 static inline void msgpack_packer_write_symbol_string_value(msgpack_packer_t* pk, VALUE v)

data/ext/msgpack/unpacker.c

@@ -142,33 +142,17 @@ static inline void reset_head_byte(msgpack_unpacker_t* uk)
 
 static inline int object_complete(msgpack_unpacker_t* uk, VALUE object)
 {
+    if(uk->freeze) {
+        rb_obj_freeze(object);
+    }
+
     uk->last_object = object;
     reset_head_byte(uk);
     return PRIMITIVE_OBJECT_COMPLETE;
 }
 
-static inline int object_complete_string(msgpack_unpacker_t* uk, VALUE str)
-{
-#ifdef COMPAT_HAVE_ENCODING
-    ENCODING_SET(str, msgpack_rb_encindex_utf8);
-#endif
-    return object_complete(uk, str);
-}
-
-static inline int object_complete_binary(msgpack_unpacker_t* uk, VALUE str)
-{
-#ifdef COMPAT_HAVE_ENCODING
-    ENCODING_SET(str, msgpack_rb_encindex_ascii8bit);
-#endif
-    return object_complete(uk, str);
-}
-
 static inline int object_complete_ext(msgpack_unpacker_t* uk, int ext_type, VALUE str)
 {
-#ifdef COMPAT_HAVE_ENCODING
-    ENCODING_SET(str, msgpack_rb_encindex_ascii8bit);
-#endif
-
     VALUE proc = msgpack_unpacker_ext_registry_lookup(&uk->ext_registry, ext_type);
     if(proc != Qnil) {
         VALUE obj = rb_funcall(proc, s_call, 1, str);
@@ -271,9 +255,10 @@ static int read_raw_body_cont(msgpack_unpacker_t* uk)
 
     int ret;
     if(uk->reading_raw_type == RAW_TYPE_STRING) {
-        ret = object_complete_string(uk, uk->reading_raw);
-    } else if(uk->reading_raw_type == RAW_TYPE_BINARY) {
-        ret = object_complete_binary(uk, uk->reading_raw);
+        ENCODING_SET(uk->reading_raw, msgpack_rb_encindex_utf8);
+        ret = object_complete(uk, uk->reading_raw);
+    } else if (uk->reading_raw_type == RAW_TYPE_BINARY) {
+        ret = object_complete(uk, uk->reading_raw);
     } else {
         ret = object_complete_ext(uk, uk->reading_raw_type, uk->reading_raw);
     }
@@ -290,13 +275,11 @@ static inline int read_raw_body_begin(msgpack_unpacker_t* uk, int raw_type)
     if(length <= msgpack_buffer_top_readable_size(UNPACKER_BUFFER_(uk))) {
         /* don't use zerocopy for hash keys but get a frozen string directly
          * because rb_hash_aset freezes keys and it causes copying */
-        bool will_freeze = is_reading_map_key(uk);
-        VALUE string = msgpack_buffer_read_top_as_string(UNPACKER_BUFFER_(uk), length, will_freeze);
+        bool will_freeze = uk->freeze || is_reading_map_key(uk);
+        VALUE string = msgpack_buffer_read_top_as_string(UNPACKER_BUFFER_(uk), length, will_freeze, raw_type == RAW_TYPE_STRING);
         int ret;
-        if(raw_type == RAW_TYPE_STRING) {
-            ret = object_complete_string(uk, string);
-        } else if(raw_type == RAW_TYPE_BINARY) {
-            ret = object_complete_binary(uk, string);
+        if(raw_type == RAW_TYPE_STRING || raw_type == RAW_TYPE_BINARY) {
+            ret = object_complete(uk, string);
         } else {
             ret = object_complete_ext(uk, raw_type, string);
         }
@@ -335,7 +318,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
     SWITCH_RANGE(b, 0xa0, 0xbf)  // FixRaw / fixstr
         int count = b & 0x1f;
         if(count == 0) {
-            return object_complete_string(uk, rb_str_buf_new(0));
+            return object_complete(uk, rb_utf8_str_new_static("", 0));
         }
         /* read_raw_body_begin sets uk->reading_raw */
         uk->reading_raw_remaining = count;
@@ -520,7 +503,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
                 uint8_t count = cb->u8;
                 if(count == 0) {
-                    return object_complete_string(uk, rb_str_buf_new(0));
+                    return object_complete(uk, rb_utf8_str_new_static("", 0));
                 }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
@@ -532,7 +515,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
                 uint16_t count = _msgpack_be16(cb->u16);
                 if(count == 0) {
-                    return object_complete_string(uk, rb_str_buf_new(0));
+                    return object_complete(uk, rb_utf8_str_new_static("", 0));
                 }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
@@ -544,7 +527,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 uint32_t count = _msgpack_be32(cb->u32);
                 if(count == 0) {
-                    return object_complete_string(uk, rb_str_buf_new(0));
+                    return object_complete(uk, rb_utf8_str_new_static("", 0));
                 }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
@@ -556,7 +539,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 1);
                 uint8_t count = cb->u8;
                 if(count == 0) {
-                    return object_complete_binary(uk, rb_str_buf_new(0));
+                    return object_complete(uk, rb_str_new_static("", 0));
                 }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
@@ -568,7 +551,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 2);
                 uint16_t count = _msgpack_be16(cb->u16);
                 if(count == 0) {
-                    return object_complete_binary(uk, rb_str_buf_new(0));
+                    return object_complete(uk, rb_str_new_static("", 0));
                 }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;
@@ -580,7 +563,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 READ_CAST_BLOCK_OR_RETURN_EOF(cb, uk, 4);
                 uint32_t count = _msgpack_be32(cb->u32);
                 if(count == 0) {
-                    return object_complete_binary(uk, rb_str_buf_new(0));
+                    return object_complete(uk, rb_str_new_static("", 0));
                 }
                 /* read_raw_body_begin sets uk->reading_raw */
                 uk->reading_raw_remaining = count;

data/ext/msgpack/unpacker.h

@@ -64,6 +64,7 @@ struct msgpack_unpacker_t {
 
     /* options */
     bool symbolize_keys;
+    bool freeze;
     bool allow_unknown_ext;
 };
 
@@ -96,6 +97,11 @@ static inline void msgpack_unpacker_set_symbolized_keys(msgpack_unpacker_t* uk,
     uk->symbolize_keys = enable;
 }
 
+static inline void msgpack_unpacker_set_freeze(msgpack_unpacker_t* uk, bool enable)
+{
+    uk->freeze = enable;
+}
+
 static inline void msgpack_unpacker_set_allow_unknown_ext(msgpack_unpacker_t* uk, bool enable)
 {
     uk->allow_unknown_ext = enable;

data/ext/msgpack/unpacker_class.c

@@ -105,6 +105,9 @@ VALUE MessagePack_Unpacker_initialize(int argc, VALUE* argv, VALUE self)
         v = rb_hash_aref(options, ID2SYM(rb_intern("symbolize_keys")));
         msgpack_unpacker_set_symbolized_keys(uk, RTEST(v));
 
+        v = rb_hash_aref(options, ID2SYM(rb_intern("freeze")));
+        msgpack_unpacker_set_freeze(uk, RTEST(v));
+
         v = rb_hash_aref(options, ID2SYM(rb_intern("allow_unknown_ext")));
         msgpack_unpacker_set_allow_unknown_ext(uk, RTEST(v));
     }
@@ -118,6 +121,12 @@ static VALUE Unpacker_symbolized_keys_p(VALUE self)
     return uk->symbolize_keys ? Qtrue : Qfalse;
 }
 
+static VALUE Unpacker_freeze_p(VALUE self)
+{
+    UNPACKER(self, uk);
+    return uk->freeze ? Qtrue : Qfalse;
+}
+
 static VALUE Unpacker_allow_unknown_ext_p(VALUE self)
 {
     UNPACKER(self, uk);
@@ -438,6 +447,7 @@ void MessagePack_Unpacker_module_init(VALUE mMessagePack)
 
     rb_define_method(cMessagePack_Unpacker, "initialize", MessagePack_Unpacker_initialize, -1);
     rb_define_method(cMessagePack_Unpacker, "symbolize_keys?", Unpacker_symbolized_keys_p, 0);
+    rb_define_method(cMessagePack_Unpacker, "freeze?", Unpacker_freeze_p, 0);
     rb_define_method(cMessagePack_Unpacker, "allow_unknown_ext?", Unpacker_allow_unknown_ext_p, 0);
     rb_define_method(cMessagePack_Unpacker, "buffer", Unpacker_buffer, 0);
     rb_define_method(cMessagePack_Unpacker, "read", Unpacker_read, 0);

data/lib/msgpack/version.rb

@@ -1,10 +1,6 @@
 module MessagePack
-  VERSION = "1.4.0.pre1"
-
-  # NOTE for msgpack-ruby maintainer:
-  # Check these things to release new binaryes for new Ruby versions (especially for Windows):
-  # * versions/supports of rake-compiler & rake-compiler-dock
-  #   https://github.com/rake-compiler/rake-compiler-dock/blob/master/History.md
-  # * update RUBY_CC_VERSION in Rakefile
-  # * check Ruby dependency of released mswin gem details
+  VERSION = "1.4.2"
+  # Note for maintainers:
+  #  Don't miss building/releasing the JRuby version (rake buld:java)
+  #  See "How to build -java rubygems" in README for more details.
 end

data/msgpack.gemspec

@@ -20,13 +20,11 @@ Gem::Specification.new do |s|
   end
   s.test_files = `git ls-files -- {test,spec}/*`.split("\n")
 
+  s.required_ruby_version = ">= 2.4"
+
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'rake'
-  s.add_development_dependency 'rake-compiler', ['~> 1.0']
-  if /java/ !~ RUBY_PLATFORM
-    # NOTE: rake-compiler-dock SHOULD be updated for new Ruby versions
-    s.add_development_dependency 'rake-compiler-dock', ['~> 1.0']
-  end
+  s.add_development_dependency 'rake-compiler'
   s.add_development_dependency 'rspec', ['~> 3.3']
   s.add_development_dependency 'yard'
   s.add_development_dependency 'json'

data/spec/spec_helper.rb

@@ -30,6 +30,12 @@ def automatic_string_keys_deduplication?
   x.keys[0].equal?(h.keys[0])
 end
 
+def string_deduplication?
+  r1 = rand.to_s
+  r2 = r1.dup
+  (-r1).equal?(-r2)
+end
+
 if java?
   RSpec.configure do |c|
     c.treat_symbols_as_metadata_keys_with_true_values = true

data/spec/unpacker_spec.rb

@@ -18,10 +18,12 @@ describe MessagePack::Unpacker do
   it 'gets options to specify how to unpack values' do
     u1 = MessagePack::Unpacker.new
     u1.symbolize_keys?.should == false
+    u1.freeze?.should == false
     u1.allow_unknown_ext?.should == false
 
-    u2 = MessagePack::Unpacker.new(symbolize_keys: true, allow_unknown_ext: true)
+    u2 = MessagePack::Unpacker.new(symbolize_keys: true, freeze: true, allow_unknown_ext: true)
     u2.symbolize_keys?.should == true
+    u2.freeze?.should == true
     u2.allow_unknown_ext?.should == true
   end
 
@@ -632,6 +634,14 @@ describe MessagePack::Unpacker do
       array = ['foo'] * 10_000
       MessagePack.unpack(MessagePack.pack(array)).size.should == 10_000
     end
+
+    it 'preserve string encoding (issue #200)' do
+      string = 'a'.force_encoding(Encoding::UTF_8)
+      MessagePack.unpack(MessagePack.pack(string)).encoding.should == string.encoding
+
+      string *= 256
+      MessagePack.unpack(MessagePack.pack(string)).encoding.should == string.encoding
+    end
   end
 
   context 'extensions' do
@@ -662,6 +672,88 @@ describe MessagePack::Unpacker do
       end
     end
 
+    context 'freeze' do
+      let :struct do
+        {'hello' => 'world', 'nested' => ['object', {'structure' => true}]}
+      end
+
+      let :buffer do
+        MessagePack.pack(struct)
+      end
+
+      let :unpacker do
+        described_class.new(:freeze => true)
+      end
+
+      it 'can freeze objects when using .unpack' do
+        parsed_struct = MessagePack.unpack(buffer, freeze: true)
+        parsed_struct.should == struct
+
+        parsed_struct.should be_frozen
+        parsed_struct['hello'].should be_frozen
+        parsed_struct['nested'].should be_frozen
+        parsed_struct['nested'][0].should be_frozen
+        parsed_struct['nested'][1].should be_frozen
+
+        if string_deduplication?
+          parsed_struct.keys[0].should be_equal('hello'.freeze)
+          parsed_struct.keys[1].should be_equal('nested'.freeze)
+          parsed_struct.values[0].should be_equal('world'.freeze)
+          parsed_struct.values[1][0].should be_equal('object'.freeze)
+          parsed_struct.values[1][1].keys[0].should be_equal('structure'.freeze)
+        end
+      end
+
+      it 'can freeze objects when using #each' do
+        objs = []
+        unpacker.feed(buffer)
+        unpacker.each do |obj|
+          objs << obj
+        end
+
+        parsed_struct = objs.first
+        parsed_struct.should == struct
+
+        parsed_struct.should be_frozen
+        parsed_struct['hello'].should be_frozen
+        parsed_struct['nested'].should be_frozen
+        parsed_struct['nested'][0].should be_frozen
+        parsed_struct['nested'][1].should be_frozen
+
+        if string_deduplication?
+          parsed_struct.keys[0].should be_equal('hello'.freeze)
+          parsed_struct.keys[1].should be_equal('nested'.freeze)
+          parsed_struct.values[0].should be_equal('world'.freeze)
+          parsed_struct.values[1][0].should be_equal('object'.freeze)
+          parsed_struct.values[1][1].keys[0].should be_equal('structure'.freeze)
+        end
+      end
+
+      it 'can freeze objects when using #feed_each' do
+        objs = []
+        unpacker.feed_each(buffer) do |obj|
+          objs << obj
+        end
+
+        parsed_struct = objs.first
+        parsed_struct.should == struct
+
+        parsed_struct.should be_frozen
+        parsed_struct['hello'].should be_frozen
+        parsed_struct['nested'].should be_frozen
+        parsed_struct['nested'][0].should be_frozen
+        parsed_struct['nested'][1].should be_frozen
+
+        if string_deduplication?
+          parsed_struct.keys[0].should be_equal('hello'.freeze)
+          parsed_struct.keys[1].should be_equal('nested'.freeze)
+          parsed_struct.values[0].should be_equal('world'.freeze)
+          parsed_struct.values[1][0].should be_equal('object'.freeze)
+          parsed_struct.values[1][1].keys[0].should be_equal('structure'.freeze)
+        end
+      end
+    end
+
     context 'binary encoding', :encodings do
       let :buffer do
         MessagePack.pack({'hello' => 'world', 'nested' => ['object', {'structure' => true}]})