msgpack 1.4.4 → 1.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c57015d343570a2bccf9f0cb8877bf7e0a83cffa478fa4a99816dc1bb4e5f3d0
-  data.tar.gz: 13e5f99e0b73c98071e8b0bb8bf841a52283ef9f1502ea578732a4baf1f4d9de
+  metadata.gz: a51f64e1c017cafcef6fe080992135f5c610984ecd126f4a3a2127baa4e4a644
+  data.tar.gz: bf45726e8f6cb658390199ecb51345e034c908207b090d53934f037555d7c37a
 SHA512:
-  metadata.gz: 2c048635dc9d9451ace0aff991a3ea92ccd7bbc397cfa4431c6428a07149fcdf9b31c1b1cd9392aca806669839236b26f90234cc99b8232061b3441707adc409
-  data.tar.gz: 63af2e6f9f11f8a96115d81aa9d8754220b44cdda069c0094ac5c635c01df75e84ac1e76dd3c2e9e81c7b3d6f5a1ba66f9006c620ec30ef047bfc7c857ffec0d
+  metadata.gz: 0ee24ab0cb11c648e9a022f23053c118c020b7c79cac14e00a1bb82faf90c59302727a2dc691c63a959225b06a5c40ec5e09fd8f35154c7474b1e652e27b9bc5
+  data.tar.gz: 2fee550949875cc119f1865b9519c75794124ec17065b1da1ec0fd06af29ed7fbdd37d709bf8cdb217d3cf86e3004a18fab16a6ac032eec176735725439ec268

data/.github/workflows/ci.yaml

@@ -12,6 +12,7 @@ on:
 jobs:
   mri:
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu, macos, windows]
         ruby: ['2.4', '2.5', '2.6', '2.7', '3.0', '3.1']
@@ -26,11 +27,10 @@ jobs:
 
   jruby:
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu]
-        # TODO: update to 9.3.3.0 once supported
-        # https://github.com/ruby/setup-ruby#supported-versions
-        ruby: ['jruby-9.2.19.0', 'jruby-9.3.2.0']
+        ruby: ['jruby-9.2.19.0', 'jruby-9.3.3.0']
     runs-on: ${{ matrix.os }}-latest
     steps:
       - uses: actions/checkout@v2
@@ -43,6 +43,7 @@ jobs:
   head-versions:
     continue-on-error: true
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu]
         ruby: ['ruby-head', 'jruby-head', 'truffleruby']

data/ChangeLog

@@ -1,3 +1,11 @@
+2022-02-15 version 1.4.5:
+
+* Fix to create UTF-8 Symbol keys when symbolize_keys: true
+* Fix to assume Symbols as US-ASCII or UTF-8
+* Optimize Packer/Unpacker initialization
+* Optimize extension class lookup
+* Rename Packer#clear as Packer#reset (#clear is still available as an alias)
+
 2022-01-22 version 1.4.4:
 
 * Specify the build option --platform=8 for older Java platforms

data/doclib/msgpack/factory.rb

@@ -63,7 +63,7 @@ module MessagePack
 
     #
     # Register a type and Class to be registered for packer and/or unpacker.
-    # If options are not speicified, factory will use :to_msgpack_ext for packer, and
+    # If options are not specified, factory will use :to_msgpack_ext for packer, and
     # :from_msgpack_ext for unpacker.
     #
     # @param type [Fixnum] type id of registered Class (0-127)

data/doclib/msgpack/packer.rb

@@ -14,7 +14,7 @@ module MessagePack
     # @overload initialize(io, options={})
     #   @param io [IO]
     #   @param options [Hash]
-    #   This packer writes serialzied objects into the IO when the internal buffer is filled.
+    #   This packer writes serialized objects into the IO when the internal buffer is filled.
     #   _io_ must respond to write(string) or append(string) method.
     #
     # Supported options:
@@ -33,12 +33,12 @@ module MessagePack
     #
     # @overload register_type(type, klass, &block)
     #   @param type [Fixnum] type id (0-127) user defined type id for specified Class
-    #   @param klass [Class] Class to be serialized with speicifed type id
+    #   @param klass [Class] Class to be serialized with specified type id
     #   @yieldparam object [Object] object to be serialized
     #
     # @overload register_type(type, klass, method_name)
     #   @param type [Fixnum] type id (0-127) user defined type id for specified Class
-    #   @param klass [Class] Class to be serialized with speicifed type id
+    #   @param klass [Class] Class to be serialized with specified type id
     #   @param method_name [Symbol] method which returns bytes of serialized representation
     #
     # @return nil
@@ -155,8 +155,9 @@ module MessagePack
     #
     # @return nil
     #
-    def clear
+    def reset
     end
+    alias clear reset
 
     #
     # Returns size of the internal buffer. Same as buffer.size.

data/doclib/msgpack/unpacker.rb

@@ -36,7 +36,7 @@ module MessagePack
     #
     # @overload register_type(type, klass, class_method_name)
     #   @param type [Fixnum] type id (0-127) user defined type id for specified Class
-    #   @param klass [Class] Class to be serialized with speicifed type id
+    #   @param klass [Class] Class to be serialized with specified type id
     #   @param class_method_name [Symbol] class method which returns an instance object
     #
     # @return nil
@@ -149,7 +149,7 @@ module MessagePack
     #
     # It repeats until the io or internal buffer does not include any complete objects.
     #
-    # If the an IO is set, it repeats to read data from the IO when the buffer
+    # If an IO is set, it repeats to read data from the IO when the buffer
     # becomes empty until the IO raises EOFError.
     #
     # This method could raise same errors with _read_ excepting EOFError.

data/ext/java/org/msgpack/jruby/ExtensionRegistry.java

@@ -82,26 +82,21 @@ public class ExtensionRegistry {
      * Objects of type Integer (Fixnum, Bignum), Float, Symbol and frozen
      * String have no singleton class and raise a TypeError when trying to get
      * it.
-     *
-     * Since all but symbols are already filtered out when reaching this code
-     * only symbols are checked here.
      */
-    if (!(object instanceof RubySymbol)) {
-      lookupClass = object.getSingletonClass();
-      pair = fetchEntryByModule(lookupClass);
-      if (pair != null) {
-          return pair;
-      }
-    }
-
-    pair = fetchEntryByModule(object.getType());
+    lookupClass = object.getMetaClass();
+    pair = fetchEntryByModule(lookupClass);
     if (pair != null) {
       return pair;
     }
 
-    if (lookupClass == null) {
-      lookupClass = object.getType(); // only for Symbol
+    RubyModule realClass = object.getType();
+    if (realClass != lookupClass) {
+      pair = fetchEntryByModule(realClass);
+      if (pair != null) {
+        return pair;
+      }
     }
+
     ExtensionEntry e = findEntryByModuleOrAncestor(lookupClass);
     if (e != null && e.hasPacker()) {
       extensionsByAncestor.put(e.getExtensionModule(), e);

data/ext/msgpack/buffer.h

@@ -494,9 +494,9 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
 #endif // HAVE_RB_ENC_INTERNED_STR
 }
 
-static inline VALUE msgpack_buffer_read_top_as_symbol(msgpack_buffer_t* b, size_t length)
+static inline VALUE msgpack_buffer_read_top_as_symbol(msgpack_buffer_t* b, size_t length, bool utf8)
 {
-    return rb_str_intern(msgpack_buffer_read_top_as_string(b, length, true, false));
+    return rb_str_intern(msgpack_buffer_read_top_as_string(b, length, true, utf8));
 }
 
 #endif

data/ext/msgpack/buffer_class.c

@@ -62,24 +62,22 @@ static VALUE Buffer_alloc(VALUE klass)
 
 static ID get_partial_read_method(VALUE io)
 {
-    if(rb_respond_to(io, s_readpartial)) {
+    if(io != Qnil && rb_respond_to(io, s_readpartial)) {
         return s_readpartial;
-    } else if(rb_respond_to(io, s_read)) {
-        return s_read;
-    } else {
-        return s_read;
     }
+    return s_read;
 }
 
 static ID get_write_all_method(VALUE io)
 {
-    if(rb_respond_to(io, s_write)) {
-        return s_write;
-    } else if(rb_respond_to(io, s_append)) {
-        return s_append;
-    } else {
-        return s_write;
+    if(io != Qnil) {
+        if(rb_respond_to(io, s_write)) {
+            return s_write;
+        } else if(rb_respond_to(io, s_append)) {
+            return s_append;
+        }
     }
+    return s_write;
 }
 
 void MessagePack_Buffer_set_options(msgpack_buffer_t* b, VALUE io, VALUE options)
@@ -245,10 +243,11 @@ static VALUE read_until_eof_rescue(VALUE args)
     return Qnil;
 }
 
-static VALUE read_until_eof_error(VALUE args)
+static VALUE read_until_eof_error(VALUE args, VALUE error)
 {
     /* ignore EOFError */
     UNUSED(args);
+    UNUSED(error);
     return Qnil;
 }
 

data/ext/msgpack/factory_class.c

@@ -30,7 +30,7 @@ typedef struct msgpack_factory_t msgpack_factory_t;
 
 struct msgpack_factory_t {
     msgpack_packer_ext_registry_t pkrg;
-    msgpack_unpacker_ext_registry_t ukrg;
+    msgpack_unpacker_ext_registry_t *ukrg;
     bool has_symbol_ext_type;
     bool optimized_symbol_ext_type;
     int symbol_ext_type;
@@ -49,14 +49,14 @@ static void Factory_free(msgpack_factory_t* fc)
         return;
     }
     msgpack_packer_ext_registry_destroy(&fc->pkrg);
-    msgpack_unpacker_ext_registry_destroy(&fc->ukrg);
+    msgpack_unpacker_ext_registry_release(fc->ukrg);
     xfree(fc);
 }
 
 void Factory_mark(msgpack_factory_t* fc)
 {
     msgpack_packer_ext_registry_mark(&fc->pkrg);
-    msgpack_unpacker_ext_registry_mark(&fc->ukrg);
+    msgpack_unpacker_ext_registry_mark(fc->ukrg);
 }
 
 static VALUE Factory_alloc(VALUE klass)
@@ -72,7 +72,7 @@ static VALUE Factory_initialize(int argc, VALUE* argv, VALUE self)
     FACTORY(self, fc);
 
     msgpack_packer_ext_registry_init(&fc->pkrg);
-    msgpack_unpacker_ext_registry_init(&fc->ukrg);
+    // fc->ukrg is lazily initialized
 
     fc->has_symbol_ext_type = false;
 
@@ -113,9 +113,7 @@ VALUE MessagePack_Factory_unpacker(int argc, VALUE* argv, VALUE self)
 
     msgpack_unpacker_t* uk;
     Data_Get_Struct(unpacker, msgpack_unpacker_t, uk);
-
-    msgpack_unpacker_ext_registry_destroy(&uk->ext_registry);
-    msgpack_unpacker_ext_registry_dup(&fc->ukrg, &uk->ext_registry);
+    msgpack_unpacker_ext_registry_borrow(fc->ukrg, &uk->ext_registry);
     uk->optimized_symbol_ext_type = fc->optimized_symbol_ext_type;
     uk->symbol_ext_type = fc->symbol_ext_type;
 
@@ -127,12 +125,19 @@ static VALUE Factory_registered_types_internal(VALUE self)
     FACTORY(self, fc);
 
     VALUE uk_mapping = rb_hash_new();
-    for(int i=0; i < 256; i++) {
-        if(fc->ukrg.array[i] != Qnil) {
-            rb_hash_aset(uk_mapping, INT2FIX(i - 128), fc->ukrg.array[i]);
+    if (fc->ukrg) {
+        for(int i=0; i < 256; i++) {
+            if(fc->ukrg->array[i] != Qnil) {
+                rb_hash_aset(uk_mapping, INT2FIX(i - 128), fc->ukrg->array[i]);
+            }
         }
     }
-    return rb_ary_new3(2, rb_hash_dup(fc->pkrg.hash), uk_mapping);
+
+    return rb_ary_new3(
+        2,
+        RTEST(fc->pkrg.hash) ? rb_hash_dup(fc->pkrg.hash) : rb_hash_new(),
+        uk_mapping
+    );
 }
 
 static VALUE Factory_register_type(int argc, VALUE* argv, VALUE self)

data/ext/msgpack/packer_class.c

@@ -281,7 +281,7 @@ static VALUE Packer_flush(VALUE self)
     return self;
 }
 
-static VALUE Packer_clear(VALUE self)
+static VALUE Packer_reset(VALUE self)
 {
     PACKER(self, pk);
     msgpack_buffer_clear(PACKER_BUFFER_(pk));
@@ -339,7 +339,10 @@ static VALUE Packer_write_to(VALUE self, VALUE io)
 static VALUE Packer_registered_types_internal(VALUE self)
 {
     PACKER(self, pk);
-    return rb_hash_dup(pk->ext_registry.hash);
+    if (RTEST(pk->ext_registry.hash)) {
+        return rb_hash_dup(pk->ext_registry.hash);
+    }
+    return rb_hash_new();
 }
 
 static VALUE Packer_register_type(int argc, VALUE* argv, VALUE self)
@@ -440,7 +443,8 @@ void MessagePack_Packer_module_init(VALUE mMessagePack)
     rb_define_method(cMessagePack_Packer, "flush", Packer_flush, 0);
 
     /* delegation methods */
-    rb_define_method(cMessagePack_Packer, "clear", Packer_clear, 0);
+    rb_define_method(cMessagePack_Packer, "reset", Packer_reset, 0);
+    rb_define_alias(cMessagePack_Packer, "clear", "reset");
     rb_define_method(cMessagePack_Packer, "size", Packer_size, 0);
     rb_define_method(cMessagePack_Packer, "empty?", Packer_empty_p, 0);
     rb_define_method(cMessagePack_Packer, "write_to", Packer_write_to, 1);

data/ext/msgpack/packer_ext_registry.c

@@ -30,8 +30,8 @@ void msgpack_packer_ext_registry_static_destroy()
 
 void msgpack_packer_ext_registry_init(msgpack_packer_ext_registry_t* pkrg)
 {
-    pkrg->hash = rb_hash_new();
-    pkrg->cache = rb_hash_new();
+    pkrg->hash = Qnil;
+    pkrg->cache = Qnil;
 }
 
 void msgpack_packer_ext_registry_mark(msgpack_packer_ext_registry_t* pkrg)
@@ -43,15 +43,20 @@ void msgpack_packer_ext_registry_mark(msgpack_packer_ext_registry_t* pkrg)
 void msgpack_packer_ext_registry_dup(msgpack_packer_ext_registry_t* src,
         msgpack_packer_ext_registry_t* dst)
 {
-    dst->hash = rb_hash_dup(src->hash);
-    dst->cache = rb_hash_dup(src->cache);
+    dst->hash = RTEST(src->hash) ? rb_hash_dup(src->hash) : Qnil;
+    dst->cache = RTEST(src->cache) ? rb_hash_dup(src->cache): Qnil;
 }
 
 VALUE msgpack_packer_ext_registry_put(msgpack_packer_ext_registry_t* pkrg,
         VALUE ext_module, int ext_type, VALUE proc, VALUE arg)
 {
-    VALUE e = rb_ary_new3(3, INT2FIX(ext_type), proc, arg);
-    /* clear lookup cache not to miss added type */
-    rb_hash_clear(pkrg->cache);
-    return rb_hash_aset(pkrg->hash, ext_module, e);
+    if (!RTEST(pkrg->hash)) {
+        pkrg->hash = rb_hash_new();
+    }
+
+    if (RTEST(pkrg->cache)) {
+        /* clear lookup cache not to miss added type */
+        rb_hash_clear(pkrg->cache);
+    }
+    return rb_hash_aset(pkrg->hash, ext_module, rb_ary_new3(3, INT2FIX(ext_type), proc, arg));
 }

data/ext/msgpack/packer_ext_registry.h

@@ -70,10 +70,12 @@ static inline VALUE msgpack_packer_ext_registry_fetch(msgpack_packer_ext_registr
     }
 
     // fetch lookup_class from cache, which stores results of searching ancestors from pkrg->hash
-    VALUE type_inht = rb_hash_lookup(pkrg->cache, lookup_class);
-    if(type_inht != Qnil) {
-        *ext_type_result = FIX2INT(rb_ary_entry(type_inht, 0));
-        return rb_ary_entry(type_inht, 1);
+    if (RTEST(pkrg->cache)) {
+        VALUE type_inht = rb_hash_lookup(pkrg->cache, lookup_class);
+        if(type_inht != Qnil) {
+            *ext_type_result = FIX2INT(rb_ary_entry(type_inht, 0));
+            return rb_ary_entry(type_inht, 1);
+        }
     }
 
     return Qnil;
@@ -82,37 +84,34 @@ static inline VALUE msgpack_packer_ext_registry_fetch(msgpack_packer_ext_registr
 static inline VALUE msgpack_packer_ext_registry_lookup(msgpack_packer_ext_registry_t* pkrg,
         VALUE instance, int* ext_type_result)
 {
-    VALUE lookup_class;
     VALUE type;
 
-    /*
-     * 1. check whether singleton_class of this instance is registered (or resolved in past) or not.
-     *
-     * Objects of type Integer (Fixnum, Bignum), Float, Symbol and frozen
-     * String have no singleton class and raise a TypeError when trying to get
-     * it. See implementation of #singleton_class in ruby's source code:
-     * VALUE rb_singleton_class(VALUE obj);
-     *
-     * Since all but symbols are already filtered out when reaching this code
-     * only symbols are checked here.
-     */
-    if (!SYMBOL_P(instance)) {
-      lookup_class = rb_singleton_class(instance);
-
-      type = msgpack_packer_ext_registry_fetch(pkrg, lookup_class, ext_type_result);
+    if (pkrg->hash == Qnil) { // No extensions registered
+        return Qnil;
+    }
 
-      if(type != Qnil) {
-          return type;
-      }
+   /*
+    * 1. check whether singleton_class or class of this instance is registered (or resolved in past) or not.
+    *
+    * Objects of type Integer (Fixnum, Bignum), Float, Symbol and frozen
+    * `rb_class_of` returns the singleton_class if the object has one, or the "real class" otherwise.
+    */
+    VALUE lookup_class = rb_class_of(instance);
+    type = msgpack_packer_ext_registry_fetch(pkrg, lookup_class, ext_type_result);
+    if(type != Qnil) {
+        return type;
     }
 
     /*
-     * 2. check the class of instance is registered (or resolved in past) or not.
+     * 2. If the object had a singleton_class check if the real class of instance is registered
+     * (or resolved in past) or not.
      */
-    type = msgpack_packer_ext_registry_fetch(pkrg, rb_obj_class(instance), ext_type_result);
-
-    if(type != Qnil) {
-        return type;
+    VALUE real_class = rb_obj_class(instance);
+    if(lookup_class != real_class) {
+        type = msgpack_packer_ext_registry_fetch(pkrg, real_class, ext_type_result);
+        if(type != Qnil) {
+            return type;
+        }
     }
 
     /*
@@ -126,6 +125,9 @@ static inline VALUE msgpack_packer_ext_registry_lookup(msgpack_packer_ext_regist
     VALUE superclass = args[1];
     if(superclass != Qnil) {
         VALUE superclass_type = rb_hash_lookup(pkrg->hash, superclass);
+        if (!RTEST(pkrg->cache)) {
+            pkrg->cache = rb_hash_new();
+        }
         rb_hash_aset(pkrg->cache, lookup_class, superclass_type);
         *ext_type_result = FIX2INT(rb_ary_entry(superclass_type, 0));
         return rb_ary_entry(superclass_type, 1);

data/ext/msgpack/unpacker.c

@@ -52,9 +52,9 @@ void msgpack_unpacker_static_destroy()
 
 #define HEAD_BYTE_REQUIRED 0xc1
 
-void _msgpack_unpacker_init(msgpack_unpacker_t* uk)
+msgpack_unpacker_t* _msgpack_unpacker_new(void)
 {
-    memset(uk, 0, sizeof(msgpack_unpacker_t));
+    msgpack_unpacker_t* uk = ZALLOC_N(msgpack_unpacker_t, 1);
 
     msgpack_buffer_init(UNPACKER_BUFFER_(uk));
 
@@ -71,6 +71,8 @@ void _msgpack_unpacker_init(msgpack_unpacker_t* uk)
     uk->stack = xmalloc(MSGPACK_UNPACKER_STACK_CAPACITY * sizeof(msgpack_unpacker_stack_t));
 #endif
     uk->stack_capacity = MSGPACK_UNPACKER_STACK_CAPACITY;
+
+    return uk;
 }
 
 void _msgpack_unpacker_destroy(msgpack_unpacker_t* uk)
@@ -164,7 +166,7 @@ static inline int object_complete_ext(msgpack_unpacker_t* uk, int ext_type, VALU
         return object_complete_symbol(uk, rb_str_intern(str));
     }
 
-    VALUE proc = msgpack_unpacker_ext_registry_lookup(&uk->ext_registry, ext_type);
+    VALUE proc = msgpack_unpacker_ext_registry_lookup(uk->ext_registry, ext_type);
     if(proc != Qnil) {
         VALUE obj = rb_funcall(proc, s_call, 1, str);
         return object_complete(uk, obj);
@@ -286,24 +288,20 @@ static inline int read_raw_body_begin(msgpack_unpacker_t* uk, int raw_type)
     if(length <= msgpack_buffer_top_readable_size(UNPACKER_BUFFER_(uk))) {
         int ret;
         if ((uk->optimized_symbol_ext_type && uk->symbol_ext_type == raw_type) || (uk->symbolize_keys && is_reading_map_key(uk))) {
-            VALUE symbol = msgpack_buffer_read_top_as_symbol(UNPACKER_BUFFER_(uk), length);
+            VALUE symbol = msgpack_buffer_read_top_as_symbol(UNPACKER_BUFFER_(uk), length, raw_type != RAW_TYPE_BINARY);
             ret = object_complete_symbol(uk, symbol);
         } else {
-            /* don't use zerocopy for hash keys but get a frozen string directly
-             * because rb_hash_aset freezes keys and it causes copying */
-            bool will_freeze = uk->freeze || is_reading_map_key(uk);
-            VALUE string = msgpack_buffer_read_top_as_string(UNPACKER_BUFFER_(uk), length, will_freeze, raw_type == RAW_TYPE_STRING);
+            bool will_freeze = uk->freeze;
             if(raw_type == RAW_TYPE_STRING || raw_type == RAW_TYPE_BINARY) {
+               /* don't use zerocopy for hash keys but get a frozen string directly
+                * because rb_hash_aset freezes keys and it causes copying */
+                will_freeze = will_freeze || is_reading_map_key(uk);
+                VALUE string = msgpack_buffer_read_top_as_string(UNPACKER_BUFFER_(uk), length, will_freeze, raw_type == RAW_TYPE_STRING);
                 ret = object_complete(uk, string);
             } else {
+                VALUE string = msgpack_buffer_read_top_as_string(UNPACKER_BUFFER_(uk), length, false, false);
                 ret = object_complete_ext(uk, raw_type, string);
             }
-
-# if !HASH_ASET_DEDUPE
-            if(will_freeze) {
-                rb_obj_freeze(string);
-            }
-# endif
         }
         uk->reading_raw_remaining = 0;
         return ret;

data/ext/msgpack/unpacker.h

@@ -60,7 +60,7 @@ struct msgpack_unpacker_t {
 
     VALUE buffer_ref;
 
-    msgpack_unpacker_ext_registry_t ext_registry;
+    msgpack_unpacker_ext_registry_t *ext_registry;
 
     /* options */
     bool symbolize_keys;
@@ -86,7 +86,7 @@ void msgpack_unpacker_static_init();
 
 void msgpack_unpacker_static_destroy();
 
-void _msgpack_unpacker_init(msgpack_unpacker_t* uk);
+msgpack_unpacker_t* _msgpack_unpacker_new(void);
 
 void _msgpack_unpacker_destroy(msgpack_unpacker_t* uk);
 

data/ext/msgpack/unpacker_class.c

@@ -45,7 +45,7 @@ static void Unpacker_free(msgpack_unpacker_t* uk)
     if(uk == NULL) {
         return;
     }
-    msgpack_unpacker_ext_registry_destroy(&uk->ext_registry);
+    msgpack_unpacker_ext_registry_release(uk->ext_registry);
     _msgpack_unpacker_destroy(uk);
     xfree(uk);
 }
@@ -53,13 +53,12 @@ static void Unpacker_free(msgpack_unpacker_t* uk)
 static void Unpacker_mark(msgpack_unpacker_t* uk)
 {
     msgpack_unpacker_mark(uk);
-    msgpack_unpacker_ext_registry_mark(&uk->ext_registry);
+    msgpack_unpacker_ext_registry_mark(uk->ext_registry);
 }
 
 VALUE MessagePack_Unpacker_alloc(VALUE klass)
 {
-    msgpack_unpacker_t* uk = ZALLOC_N(msgpack_unpacker_t, 1);
-    _msgpack_unpacker_init(uk);
+    msgpack_unpacker_t* uk = _msgpack_unpacker_new();
 
     VALUE self = Data_Wrap_Struct(klass, Unpacker_mark, Unpacker_free, uk);
     return self;
@@ -94,7 +93,6 @@ VALUE MessagePack_Unpacker_initialize(int argc, VALUE* argv, VALUE self)
 
     UNPACKER(self, uk);
 
-    msgpack_unpacker_ext_registry_init(&uk->ext_registry);
     uk->buffer_ref = MessagePack_Buffer_wrap(UNPACKER_BUFFER_(uk), self);
 
     MessagePack_Buffer_set_options(UNPACKER_BUFFER_(uk), io, options);
@@ -133,7 +131,7 @@ static VALUE Unpacker_allow_unknown_ext_p(VALUE self)
     return uk->allow_unknown_ext ? Qtrue : Qfalse;
 }
 
-static void raise_unpacker_error(int r)
+NORETURN(static void raise_unpacker_error(int r))
 {
     switch(r) {
     case PRIMITIVE_EOF:
@@ -222,34 +220,6 @@ static VALUE Unpacker_read_map_header(VALUE self)
     return ULONG2NUM(size);
 }
 
-static VALUE Unpacker_peek_next_type(VALUE self)
-{
-    UNPACKER(self, uk);
-
-    int r = msgpack_unpacker_peek_next_object_type(uk);
-    if(r < 0) {
-        raise_unpacker_error(r);
-    }
-
-    switch((enum msgpack_unpacker_object_type) r) {
-    case TYPE_NIL:
-        return rb_intern("nil");
-    case TYPE_BOOLEAN:
-        return rb_intern("boolean");
-    case TYPE_INTEGER:
-        return rb_intern("integer");
-    case TYPE_FLOAT:
-        return rb_intern("float");
-    case TYPE_RAW:
-        return rb_intern("raw");
-    case TYPE_ARRAY:
-        return rb_intern("array");
-    case TYPE_MAP:
-        return rb_intern("map");
-    default:
-        rb_raise(eUnpackError, "logically unknown type %d", r);
-    }
-}
 
 static VALUE Unpacker_feed(VALUE self, VALUE data)
 {
@@ -296,9 +266,10 @@ static VALUE Unpacker_each_impl(VALUE self)
     }
 }
 
-static VALUE Unpacker_rescue_EOFError(VALUE self)
+static VALUE Unpacker_rescue_EOFError(VALUE args, VALUE error)
 {
-    UNUSED(self);
+    UNUSED(args);
+    UNUSED(error);
     return Qnil;
 }
 
@@ -347,9 +318,11 @@ static VALUE Unpacker_registered_types_internal(VALUE self)
     UNPACKER(self, uk);
 
     VALUE mapping = rb_hash_new();
-    for(int i=0; i < 256; i++) {
-        if(uk->ext_registry.array[i] != Qnil) {
-            rb_hash_aset(mapping, INT2FIX(i - 128), uk->ext_registry.array[i]);
+    if (uk->ext_registry) {
+        for(int i=0; i < 256; i++) {
+            if(uk->ext_registry->array[i] != Qnil) {
+                rb_hash_aset(mapping, INT2FIX(i - 128), uk->ext_registry->array[i]);
+            }
         }
     }
 
@@ -451,7 +424,6 @@ void MessagePack_Unpacker_module_init(VALUE mMessagePack)
     rb_define_method(cMessagePack_Unpacker, "skip_nil", Unpacker_skip_nil, 0);
     rb_define_method(cMessagePack_Unpacker, "read_array_header", Unpacker_read_array_header, 0);
     rb_define_method(cMessagePack_Unpacker, "read_map_header", Unpacker_read_map_header, 0);
-    //rb_define_method(cMessagePack_Unpacker, "peek_next_type", Unpacker_peek_next_type, 0);  // TODO
     rb_define_method(cMessagePack_Unpacker, "feed", Unpacker_feed, 1);
     rb_define_method(cMessagePack_Unpacker, "feed_reference", Unpacker_feed_reference, 1);
     rb_define_method(cMessagePack_Unpacker, "each", Unpacker_each, 0);

data/ext/msgpack/unpacker_ext_registry.c

@@ -27,36 +27,59 @@ void msgpack_unpacker_ext_registry_static_init()
     s_dup = rb_intern("dup");
 }
 
+
 void msgpack_unpacker_ext_registry_static_destroy()
 { }
 
-void msgpack_unpacker_ext_registry_init(msgpack_unpacker_ext_registry_t* ukrg)
+void msgpack_unpacker_ext_registry_mark(msgpack_unpacker_ext_registry_t* ukrg)
 {
-    for(int i=0; i < 256; i++) {
-        ukrg->array[i] = Qnil;
+    if (ukrg) {
+        for(int i=0; i < 256; i++) {
+            if (ukrg->array[i] != Qnil) {
+                rb_gc_mark(ukrg->array[i]);
+            }
+        }
     }
 }
 
-void msgpack_unpacker_ext_registry_mark(msgpack_unpacker_ext_registry_t* ukrg)
+msgpack_unpacker_ext_registry_t* msgpack_unpacker_ext_registry_cow(msgpack_unpacker_ext_registry_t* src)
 {
-    for(int i=0; i < 256; i++) {
-        rb_gc_mark(ukrg->array[i]);
+    msgpack_unpacker_ext_registry_t* dst;
+    if (src) {
+        if (src->borrow_count) {
+            dst = ALLOC(msgpack_unpacker_ext_registry_t);
+            dst->borrow_count = 0;
+            MEMCPY(dst->array, src->array, VALUE, 256);
+            msgpack_unpacker_ext_registry_release(src);
+            return dst;
+        } else {
+            return src;
+        }
+    } else {
+        dst = ALLOC(msgpack_unpacker_ext_registry_t);
+        dst->borrow_count = 0;
+        for(int i=0; i < 256; i++) {
+            dst->array[i] = Qnil;
+        }
+        return dst;
     }
 }
 
-void msgpack_unpacker_ext_registry_dup(msgpack_unpacker_ext_registry_t* src,
-        msgpack_unpacker_ext_registry_t* dst)
+void msgpack_unpacker_ext_registry_release(msgpack_unpacker_ext_registry_t* ukrg)
 {
-    for(int i=0; i < 256; i++) {
-        dst->array[i] = src->array[i];
+    if (ukrg) {
+        if (ukrg->borrow_count) {
+            ukrg->borrow_count--;
+        } else {
+            xfree(ukrg);
+        }
     }
 }
 
-VALUE msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t* ukrg,
+void msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t** ukrg,
         VALUE ext_module, int ext_type, VALUE proc, VALUE arg)
 {
-    VALUE e = rb_ary_new3(3, ext_module, proc, arg);
-    VALUE before = ukrg->array[ext_type + 128];
-    ukrg->array[ext_type + 128] = e;
-    return before;
+    msgpack_unpacker_ext_registry_t* ext_registry = msgpack_unpacker_ext_registry_cow(*ukrg);
+    ext_registry->array[ext_type + 128] = rb_ary_new3(3, ext_module, proc, arg);
+    *ukrg = ext_registry;
 }

data/ext/msgpack/unpacker_ext_registry.h

@@ -25,35 +25,39 @@ struct msgpack_unpacker_ext_registry_t;
 typedef struct msgpack_unpacker_ext_registry_t msgpack_unpacker_ext_registry_t;
 
 struct msgpack_unpacker_ext_registry_t {
+    unsigned int borrow_count;
     VALUE array[256];
-    //int bitmap;
 };
 
 void msgpack_unpacker_ext_registry_static_init();
 
 void msgpack_unpacker_ext_registry_static_destroy();
 
-void msgpack_unpacker_ext_registry_init(msgpack_unpacker_ext_registry_t* ukrg);
+void msgpack_unpacker_ext_registry_release(msgpack_unpacker_ext_registry_t* ukrg);
 
-static inline void msgpack_unpacker_ext_registry_destroy(msgpack_unpacker_ext_registry_t* ukrg)
-{ }
+static inline void msgpack_unpacker_ext_registry_borrow(msgpack_unpacker_ext_registry_t* src, msgpack_unpacker_ext_registry_t** dst)
+{
+    if (src) {
+        src->borrow_count++;
+        *dst = src;
+    }
+}
 
 void msgpack_unpacker_ext_registry_mark(msgpack_unpacker_ext_registry_t* ukrg);
 
-void msgpack_unpacker_ext_registry_dup(msgpack_unpacker_ext_registry_t* src,
-        msgpack_unpacker_ext_registry_t* dst);
-
-VALUE msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t* ukrg,
+void msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t** ukrg,
         VALUE ext_module, int ext_type, VALUE proc, VALUE arg);
 
 static inline VALUE msgpack_unpacker_ext_registry_lookup(msgpack_unpacker_ext_registry_t* ukrg,
         int ext_type)
 {
-    VALUE e = ukrg->array[ext_type + 128];
-    if(e == Qnil) {
-        return Qnil;
+    if (ukrg) {
+        VALUE entry = ukrg->array[ext_type + 128];
+        if (entry != Qnil) {
+            return rb_ary_entry(entry, 1);
+        }
     }
-    return rb_ary_entry(e, 1);
+    return Qnil;
 }
 
 #endif

data/lib/msgpack/symbol.rb

@@ -14,6 +14,13 @@ class Symbol
     # The canonical way to do it for symbols would be:
     #  data.unpack1('A*').to_sym
     # However in this instance we can take a shortcut
-    data.to_sym
+
+    # We assume the string encoding is UTF-8, and let Ruby create either
+    # an ASCII symbol or UTF-8 symbol.
+    data.force_encoding(Encoding::UTF_8).to_sym
+  rescue EncodingError
+    # If somehow the string wasn't valid UTF-8 not valid ASCII, we fallback
+    # to what has been the historical behavior of creating a binary symbol
+    data.force_encoding(Encoding::BINARY).to_sym
   end
 end

data/lib/msgpack/version.rb

@@ -1,5 +1,5 @@
 module MessagePack
-  VERSION = "1.4.4"
+  VERSION = "1.4.5"
   # Note for maintainers:
   #  Don't miss building/releasing the JRuby version (rake buld:java)
   #  See "How to build -java rubygems" in README for more details.

data/spec/factory_spec.rb

@@ -1,4 +1,3 @@
-# encoding: ascii-8bit
 require 'spec_helper'
 
 describe MessagePack::Factory do
@@ -42,6 +41,21 @@ describe MessagePack::Factory do
       unpacker.feed(MessagePack::ExtensionValue.new(1, 'a').to_msgpack)
       expect{ unpacker.read }.to raise_error(MessagePack::UnknownExtTypeError)
     end
+
+    it 'does not share the extension registry with unpackers' do
+      subject.register_type(0x00, Symbol)
+      expect do
+        unpacker = subject.unpacker
+        expect do
+          unpacker.register_type(0x01) {}
+        end.to change { unpacker.registered_types }
+
+        second_unpacker = subject.unpacker
+        expect do
+          second_unpacker.register_type(0x01) {}
+        end.to_not change { unpacker.registered_types }
+      end.to_not change { subject.registered_types }
+    end
   end
 
   describe '#dump and #load' do
@@ -255,29 +269,25 @@ describe MessagePack::Factory do
         subject { factory.packer.pack(value).to_s }
         before { stub_const('Value', Class.new{ include Mod }) }
         let(:value) { Value.new }
-        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
+        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked".force_encoding(Encoding::BINARY) }
       end
 
       describe "packing an object which has been extended by the module" do
         subject { factory.packer.pack(object).to_s }
         let(:object) { Object.new.extend Mod }
-        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
+        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked".force_encoding(Encoding::BINARY) }
       end
 
       describe "unpacking with the module" do
-        subject { factory.unpacker.feed("\xC7\x06\x01module").unpack }
+        subject { factory.unpacker.feed("\xC7\x06\x01module".force_encoding(Encoding::BINARY)).unpack }
         it { is_expected.to eq "unpacked module" }
       end
     end
   end
 
   describe 'the special treatment of symbols with ext type' do
-    let(:packer) { subject.packer }
-    let(:unpacker) { subject.unpacker }
-
-    def symbol_after_roundtrip
-      packed_symbol = packer.pack(:symbol).to_s
-      unpacker.feed(packed_symbol).unpack
+    def roundtrip(object, options = nil)
+      subject.load(subject.dump(object), options)
     end
 
     context 'using the optimized symbol unpacker' do
@@ -293,13 +303,25 @@ describe MessagePack::Factory do
       end
 
       it 'lets symbols survive a roundtrip' do
-        expect(symbol_after_roundtrip).to be :symbol
+        expect(roundtrip(:symbol)).to be :symbol
+      end
+
+      it 'preserves encoding for ASCII symbols' do
+        expect(:symbol.encoding).to be Encoding::US_ASCII
+        expect(roundtrip(:symbol)).to be :symbol
+        expect(roundtrip(:symbol).encoding).to be Encoding::US_ASCII
+      end
+
+      it 'preserves encoding for UTF-8 symbols' do
+        expect(:"fée".encoding).to be Encoding::UTF_8
+        expect(roundtrip(:"fée").encoding).to be Encoding::UTF_8
+        expect(roundtrip(:"fée")).to be :"fée"
       end
     end
 
     context 'if no ext type is registered for symbols' do
       it 'converts symbols to string' do
-        expect(symbol_after_roundtrip).to eq 'symbol'
+        expect(roundtrip(:symbol)).to eq 'symbol'
       end
     end
 
@@ -308,7 +330,41 @@ describe MessagePack::Factory do
         before { subject.register_type(0x00, ::Symbol) }
 
         it 'lets symbols survive a roundtrip' do
-          expect(symbol_after_roundtrip).to be :symbol
+          expect(roundtrip(:symbol)).to be :symbol
+        end
+
+        it 'works with hash keys' do
+          expect(roundtrip(symbol: 1)).to be == { symbol: 1 }
+        end
+
+        it 'works with frozen: true option' do
+          expect(roundtrip(:symbol, freeze: true)).to be :symbol
+        end
+
+        it 'preserves encoding for ASCII symbols' do
+          expect(:symbol.encoding).to be Encoding::US_ASCII
+          expect(roundtrip(:symbol)).to be :symbol
+          expect(roundtrip(:symbol).encoding).to be Encoding::US_ASCII
+        end
+
+        it 'preserves encoding for UTF-8 symbols' do
+          expect(:"fée".encoding).to be Encoding::UTF_8
+          expect(roundtrip(:"fée")).to be :"fée"
+          expect(roundtrip(:"fée").encoding).to be Encoding::UTF_8
+        end
+
+        it 'does not handle symbols in other encodings' do
+          symbol = "fàe".encode(Encoding::ISO_8859_1).to_sym
+          expect(symbol.encoding).to be Encoding::ISO_8859_1
+
+          if IS_JRUBY
+            # JRuby doesn't quite behave like MRI here.
+            # "fàe".force_encoding(Encoding::BINARY).to_sym is able to lookup the existing ISO-8859-1 symbol
+            # It likely is a JRuby bug.
+            expect(roundtrip(symbol).encoding).to be Encoding::ISO_8859_1
+          else
+            expect(roundtrip(symbol).encoding).to be Encoding::BINARY
+          end
         end
       end
 
@@ -332,7 +388,7 @@ describe MessagePack::Factory do
         before { subject.register_type(0x00, ::Symbol) }
 
         it 'lets symbols survive a roundtrip' do
-          expect(symbol_after_roundtrip).to be :symbol
+          expect(roundtrip(:symbol)).to be :symbol
         end
 
         after do

data/spec/spec_helper.rb

@@ -25,9 +25,7 @@ if GC.respond_to?(:auto_compact=)
   GC.auto_compact = true
 end
 
-def java?
-  /java/ =~ RUBY_PLATFORM
-end
+IS_JRUBY = RUBY_ENGINE == 'jruby'
 
 # checking if Hash#[]= (rb_hash_aset) dedupes string keys
 def automatic_string_keys_deduplication?
@@ -46,7 +44,7 @@ def string_deduplication?
   (-r1).equal?(-r2)
 end
 
-if java?
+if IS_JRUBY
   RSpec.configure do |c|
     c.treat_symbols_as_metadata_keys_with_true_values = true
     c.filter_run_excluding :encodings => !(defined? Encoding)

data/spec/timestamp_spec.rb

@@ -2,8 +2,6 @@
 
 require 'spec_helper'
 
-IS_JRUBY = Kernel.const_defined?(:JRUBY_VERSION)
-
 describe MessagePack::Timestamp do
   describe 'malformed format' do
     it do

data/spec/unpacker_spec.rb

@@ -1,5 +1,3 @@
-# encoding: ascii-8bit
-
 require 'stringio'
 require 'tempfile'
 require 'zlib'
@@ -302,6 +300,21 @@ describe MessagePack::Unpacker do
     MessagePack.unpack(MessagePack.pack(symbolized_hash), :symbolize_keys => true).should == symbolized_hash
   end
 
+  it 'MessagePack.unpack symbolize_keys preserve encoding' do
+    hash = { :ascii => 1, :utf8_é => 2}
+    loaded_hash = MessagePack.load(MessagePack.pack(hash), :symbolize_keys => true)
+
+    hash.keys[0].encoding.should == Encoding::US_ASCII # Ruby coerce symbols to US-ASCII when possible.
+    loaded_hash.keys[0].should == hash.keys[0]
+    loaded_hash.keys[0].encoding.should == hash.keys[0].encoding
+
+    hash.keys[1].encoding.should == Encoding::UTF_8
+    loaded_hash.keys[1].should == hash.keys[1]
+    loaded_hash.keys[1].encoding.should == hash.keys[1].encoding
+
+    MessagePack.unpack(MessagePack.pack(hash), :symbolize_keys => true).should == hash
+  end
+
   it 'Unpacker#unpack symbolize_keys' do
     unpacker = MessagePack::Unpacker.new(:symbolize_keys => true)
     symbolized_hash = {:a => 'b', :c => 'd'}
@@ -765,7 +778,13 @@ describe MessagePack::Unpacker do
 
     context 'binary encoding', :encodings do
       let :buffer do
-        MessagePack.pack({'hello' => 'world', 'nested' => ['object', {'structure' => true}]})
+        MessagePack.pack({
+          'hello'.b => 'world'.b,
+          'nested'.b => [
+            'object'.b,
+            {'structure'.b => true},
+          ]
+        })
       end
 
       let :unpacker do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: msgpack
 version: !ruby/object:Gem::Version
-  version: 1.4.4
+  version: 1.4.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-22 00:00:00.000000000 Z
+date: 2022-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler