msgpack 1.4.5 → 1.5.0

data/ext/java/org/msgpack/jruby/Unpacker.java

@@ -56,30 +56,48 @@ public class Unpacker extends RubyObject {
 
   @JRubyMethod(name = "initialize", optional = 2, visibility = PRIVATE)
   public IRubyObject initialize(ThreadContext ctx, IRubyObject[] args) {
+    Ruby runtime = ctx.runtime;
+
     symbolizeKeys = false;
     allowUnknownExt = false;
     freeze = false;
-    if (args.length > 0) {
-      Ruby runtime = ctx.runtime;
-      if (args[args.length - 1] instanceof RubyHash) {
-        RubyHash options = (RubyHash) args[args.length - 1];
-        IRubyObject sk = options.fastARef(runtime.newSymbol("symbolize_keys"));
-        if (sk != null) {
-          symbolizeKeys = sk.isTrue();
-        }
-        IRubyObject f = options.fastARef(runtime.newSymbol("freeze"));
-        if (f != null) {
-          freeze = f.isTrue();
-        }
-        IRubyObject au = options.fastARef(runtime.newSymbol("allow_unknown_ext"));
-        if (au != null) {
-          allowUnknownExt = au.isTrue();
-        }
+
+    IRubyObject io = null;
+    RubyHash options = null;
+
+    if (args.length >= 1) {
+      io = args[0];
+    }
+
+    if (args.length >= 2 && args[1] != runtime.getNil()) {
+      options = (RubyHash)args[1];
+    }
+
+    if (options == null && io != null && io instanceof RubyHash) {
+      options = (RubyHash)io;
+      io = null;
+    }
+
+    if (options != null) {
+      IRubyObject sk = options.fastARef(runtime.newSymbol("symbolize_keys"));
+      if (sk != null) {
+        symbolizeKeys = sk.isTrue();
       }
-      if (args[0] != runtime.getNil() && !(args[0] instanceof RubyHash)) {
-        setStream(ctx, args[0]);
+      IRubyObject f = options.fastARef(runtime.newSymbol("freeze"));
+      if (f != null) {
+        freeze = f.isTrue();
       }
+      IRubyObject au = options.fastARef(runtime.newSymbol("allow_unknown_ext"));
+      if (au != null) {
+        allowUnknownExt = au.isTrue();
+      }
+
     }
+
+    if (io != null && io != runtime.getNil()) {
+      setStream(ctx, io);
+    }
+
     return this;
   }
 
@@ -139,7 +157,7 @@ public class Unpacker extends RubyObject {
       throw runtime.newRangeError(String.format("integer %d too big to convert to `signed char'", typeId));
     }
 
-    registry.put(extModule, (int) typeId, null, null, proc, arg);
+    registry.put(extModule, (int) typeId, false, null, null, proc, arg);
     return runtime.getNil();
   }
 
@@ -157,7 +175,7 @@ public class Unpacker extends RubyObject {
     if (limit == -1) {
       limit = byteList.length() - offset;
     }
-    Decoder decoder = new Decoder(ctx.runtime, registry, byteList.unsafeBytes(), byteList.begin() + offset, limit, symbolizeKeys, freeze, allowUnknownExt);
+    Decoder decoder = new Decoder(ctx.runtime, this, byteList.unsafeBytes(), byteList.begin() + offset, limit, symbolizeKeys, freeze, allowUnknownExt);
     try {
       data = null;
       data = decoder.next();
@@ -187,7 +205,7 @@ public class Unpacker extends RubyObject {
   public IRubyObject feed(ThreadContext ctx, IRubyObject data) {
     ByteList byteList = data.asString().getByteList();
     if (decoder == null) {
-      decoder = new Decoder(ctx.runtime, registry, byteList.unsafeBytes(), byteList.begin(), byteList.length(), symbolizeKeys, freeze, allowUnknownExt);
+      decoder = new Decoder(ctx.runtime, this, byteList.unsafeBytes(), byteList.begin(), byteList.length(), symbolizeKeys, freeze, allowUnknownExt);
     } else {
       decoder.feed(byteList.unsafeBytes(), byteList.begin(), byteList.length());
     }
@@ -325,7 +343,11 @@ public class Unpacker extends RubyObject {
     ByteList byteList = str.getByteList();
     this.stream = stream;
     this.decoder = null;
-    this.decoder = new Decoder(ctx.runtime, registry, byteList.unsafeBytes(), byteList.begin(), byteList.length(), symbolizeKeys, freeze, allowUnknownExt);
+    this.decoder = new Decoder(ctx.runtime, this, byteList.unsafeBytes(), byteList.begin(), byteList.length(), symbolizeKeys, freeze, allowUnknownExt);
     return getStream(ctx);
   }
+
+  public ExtensionRegistry.ExtensionEntry lookupExtensionByTypeId(int typeId) {
+    return registry.lookupExtensionByTypeId(typeId);
+  }
 }

data/ext/msgpack/buffer_class.c

@@ -29,6 +29,11 @@ static ID s_write;
 static ID s_append;
 static ID s_close;
 
+static VALUE sym_read_reference_threshold;
+static VALUE sym_write_reference_threshold;
+static VALUE sym_io_buffer_size;
+
+
 #define BUFFER(from, name) \
     msgpack_buffer_t *name = NULL; \
     Data_Get_Struct(from, msgpack_buffer_t, name); \
@@ -89,17 +94,17 @@ void MessagePack_Buffer_set_options(msgpack_buffer_t* b, VALUE io, VALUE options
     if(options != Qnil) {
         VALUE v;
 
-        v = rb_hash_aref(options, ID2SYM(rb_intern("read_reference_threshold")));
+        v = rb_hash_aref(options, sym_read_reference_threshold);
         if(v != Qnil) {
             msgpack_buffer_set_read_reference_threshold(b, NUM2ULONG(v));
         }
 
-        v = rb_hash_aref(options, ID2SYM(rb_intern("write_reference_threshold")));
+        v = rb_hash_aref(options, sym_write_reference_threshold);
         if(v != Qnil) {
             msgpack_buffer_set_write_reference_threshold(b, NUM2ULONG(v));
         }
 
-        v = rb_hash_aref(options, ID2SYM(rb_intern("io_buffer_size")));
+        v = rb_hash_aref(options, sym_io_buffer_size);
         if(v != Qnil) {
             msgpack_buffer_set_io_buffer_size(b, NUM2ULONG(v));
         }
@@ -479,6 +484,10 @@ void MessagePack_Buffer_module_init(VALUE mMessagePack)
     s_append = rb_intern("<<");
     s_close = rb_intern("close");
 
+    sym_read_reference_threshold = ID2SYM(rb_intern("read_reference_threshold"));
+    sym_write_reference_threshold = ID2SYM(rb_intern("write_reference_threshold"));
+    sym_io_buffer_size = ID2SYM(rb_intern("io_buffer_size"));
+
     msgpack_buffer_static_init();
 
     cMessagePack_Buffer = rb_define_class_under(mMessagePack, "Buffer", rb_cObject);

data/ext/msgpack/factory_class.c

@@ -31,6 +31,7 @@ typedef struct msgpack_factory_t msgpack_factory_t;
 struct msgpack_factory_t {
     msgpack_packer_ext_registry_t pkrg;
     msgpack_unpacker_ext_registry_t *ukrg;
+    bool has_bigint_ext_type;
     bool has_symbol_ext_type;
     bool optimized_symbol_ext_type;
     int symbol_ext_type;
@@ -87,6 +88,41 @@ static VALUE Factory_initialize(int argc, VALUE* argv, VALUE self)
     return Qnil;
 }
 
+static VALUE Factory_dup(VALUE self)
+{
+    VALUE clone = Factory_alloc(rb_obj_class(self));
+
+    FACTORY(self, fc);
+    FACTORY(clone, cloned_fc);
+
+    cloned_fc->has_symbol_ext_type = fc->has_symbol_ext_type;
+    cloned_fc->pkrg = fc->pkrg;
+    msgpack_unpacker_ext_registry_borrow(fc->ukrg, &cloned_fc->ukrg);
+    msgpack_packer_ext_registry_dup(&fc->pkrg, &cloned_fc->pkrg);
+
+    return clone;
+}
+
+static VALUE Factory_freeze(VALUE self) {
+    if(!rb_obj_frozen_p(self)) {
+        FACTORY(self, fc);
+
+        if (RTEST(fc->pkrg.hash)) {
+            rb_hash_freeze(fc->pkrg.hash);
+            if (!RTEST(fc->pkrg.cache)) {
+                // If the factory is frozen, we can safely share the packer cache between
+                // all packers. So we eagerly create it now so it's available when #packer
+                // is called.
+                fc->pkrg.cache = rb_hash_new();
+            }
+        }
+
+        rb_obj_freeze(self);
+    }
+
+    return self;
+}
+
 VALUE MessagePack_Factory_packer(int argc, VALUE* argv, VALUE self)
 {
     FACTORY(self, fc);
@@ -99,6 +135,7 @@ VALUE MessagePack_Factory_packer(int argc, VALUE* argv, VALUE self)
 
     msgpack_packer_ext_registry_destroy(&pk->ext_registry);
     msgpack_packer_ext_registry_dup(&fc->pkrg, &pk->ext_registry);
+    pk->has_bigint_ext_type = fc->has_bigint_ext_type;
     pk->has_symbol_ext_type = fc->has_symbol_ext_type;
 
     return packer;
@@ -145,6 +182,7 @@ static VALUE Factory_register_type(int argc, VALUE* argv, VALUE self)
     FACTORY(self, fc);
 
     int ext_type;
+    int flags = 0;
     VALUE ext_module;
     VALUE options = Qnil;
     VALUE packer_arg, unpacker_arg;
@@ -173,6 +211,10 @@ static VALUE Factory_register_type(int argc, VALUE* argv, VALUE self)
         rb_raise(rb_eArgError, "wrong number of arguments (%d for 2..3)", argc);
     }
 
+    if (options != Qnil) {
+        Check_Type(options, T_HASH);
+    }
+
     ext_type = NUM2INT(argv[0]);
     if(ext_type < -128 || ext_type > 127) {
         rb_raise(rb_eRangeError, "integer %d too big to convert to `signed char'", ext_type);
@@ -200,16 +242,29 @@ static VALUE Factory_register_type(int argc, VALUE* argv, VALUE self)
         }
     }
 
-    msgpack_packer_ext_registry_put(&fc->pkrg, ext_module, ext_type, packer_proc, packer_arg);
-
-    if (ext_module == rb_cSymbol) {
+    if(ext_module == rb_cSymbol) {
         fc->has_symbol_ext_type = true;
-        if(RB_TEST(options) && RB_TEST(rb_hash_aref(options, ID2SYM(rb_intern("optimized_symbols_parsing"))))) {
+        if(RTEST(options) && RTEST(rb_hash_aref(options, ID2SYM(rb_intern("optimized_symbols_parsing"))))) {
             fc->optimized_symbol_ext_type = true;
         }
     }
 
-    msgpack_unpacker_ext_registry_put(&fc->ukrg, ext_module, ext_type, unpacker_proc, unpacker_arg);
+    if(RTEST(options)) {
+        if(RTEST(rb_hash_aref(options, ID2SYM(rb_intern("oversized_integer_extension"))))) {
+            if(ext_module == rb_cInteger) {
+                fc->has_bigint_ext_type = true;
+            } else {
+                rb_raise(rb_eArgError, "oversized_integer_extension: true is only for Integer class");
+            }
+        }
+
+        if(RTEST(rb_hash_aref(options, ID2SYM(rb_intern("recursive"))))) {
+            flags |= MSGPACK_EXT_RECURSIVE;
+        }
+    }
+
+    msgpack_packer_ext_registry_put(&fc->pkrg, ext_module, ext_type, flags, packer_proc, packer_arg);
+    msgpack_unpacker_ext_registry_put(&fc->ukrg, ext_module, ext_type, flags, unpacker_proc, unpacker_arg);
 
     return Qnil;
 }
@@ -221,6 +276,8 @@ void MessagePack_Factory_module_init(VALUE mMessagePack)
     rb_define_alloc_func(cMessagePack_Factory, Factory_alloc);
 
     rb_define_method(cMessagePack_Factory, "initialize", Factory_initialize, -1);
+    rb_define_method(cMessagePack_Factory, "dup", Factory_dup, 0);
+    rb_define_method(cMessagePack_Factory, "freeze", Factory_freeze, 0);
 
     rb_define_method(cMessagePack_Factory, "packer", MessagePack_Factory_packer, -1);
     rb_define_method(cMessagePack_Factory, "unpacker", MessagePack_Factory_unpacker, -1);

data/ext/msgpack/packer.c

@@ -121,19 +121,56 @@ void msgpack_packer_write_hash_value(msgpack_packer_t* pk, VALUE v)
 #endif
 }
 
+struct msgpack_call_proc_args_t;
+typedef struct msgpack_call_proc_args_t msgpack_call_proc_args_t;
+struct msgpack_call_proc_args_t {
+    VALUE proc;
+    VALUE arg;
+    VALUE packer;
+};
+
+VALUE msgpack_packer_try_calling_proc(VALUE value)
+{
+    msgpack_call_proc_args_t *args = (msgpack_call_proc_args_t *)value;
+    return rb_funcall(args->proc, s_call, 2, args->arg, args->packer);
+}
+
 bool msgpack_packer_try_write_with_ext_type_lookup(msgpack_packer_t* pk, VALUE v)
 {
-    int ext_type;
+    int ext_type, ext_flags;
 
-    VALUE proc = msgpack_packer_ext_registry_lookup(&pk->ext_registry, v, &ext_type);
+    VALUE proc = msgpack_packer_ext_registry_lookup(&pk->ext_registry, v, &ext_type, &ext_flags);
 
-    if(proc != Qnil) {
+    if(proc == Qnil) {
+        return false;
+    }
+
+    if(ext_flags & MSGPACK_EXT_RECURSIVE) {
+        msgpack_buffer_t parent_buffer = pk->buffer;
+        msgpack_buffer_init(PACKER_BUFFER_(pk));
+
+        int exception_occured = 0;
+        msgpack_call_proc_args_t args = { proc, v, pk->to_msgpack_arg };
+        rb_protect(msgpack_packer_try_calling_proc, (VALUE)&args, &exception_occured);
+
+        if (exception_occured) {
+            msgpack_buffer_destroy(PACKER_BUFFER_(pk));
+            pk->buffer = parent_buffer;
+            rb_jump_tag(exception_occured); // re-raise the exception
+        } else {
+            VALUE payload = msgpack_buffer_all_as_string(PACKER_BUFFER_(pk));
+            StringValue(payload);
+            msgpack_buffer_destroy(PACKER_BUFFER_(pk));
+            pk->buffer = parent_buffer;
+            msgpack_packer_write_ext(pk, ext_type, payload);
+        }
+    } else {
         VALUE payload = rb_funcall(proc, s_call, 1, v);
         StringValue(payload);
         msgpack_packer_write_ext(pk, ext_type, payload);
-        return true;
     }
-    return false;
+
+    return true;
 }
 
 void msgpack_packer_write_other_value(msgpack_packer_t* pk, VALUE v)

data/ext/msgpack/packer.h

@@ -32,6 +32,7 @@ struct msgpack_packer_t {
     msgpack_buffer_t buffer;
 
     bool compatibility_mode;
+    bool has_bigint_ext_type;
     bool has_symbol_ext_type;
 
     ID to_msgpack_method;
@@ -56,6 +57,8 @@ void msgpack_packer_destroy(msgpack_packer_t* pk);
 
 void msgpack_packer_mark(msgpack_packer_t* pk);
 
+bool msgpack_packer_try_write_with_ext_type_lookup(msgpack_packer_t* pk, VALUE v);
+
 static inline void msgpack_packer_set_to_msgpack_method(msgpack_packer_t* pk,
         ID to_msgpack_method, VALUE to_msgpack_arg)
 {
@@ -469,9 +472,30 @@ static inline void msgpack_packer_write_fixnum_value(msgpack_packer_t* pk, VALUE
 
 static inline void msgpack_packer_write_bignum_value(msgpack_packer_t* pk, VALUE v)
 {
+    int leading_zero_bits;
+    size_t required_size = rb_absint_size(v, &leading_zero_bits);
+
     if(RBIGNUM_POSITIVE_P(v)) {
+        if(required_size > 8 && pk->has_bigint_ext_type) {
+            if(msgpack_packer_try_write_with_ext_type_lookup(pk, v)) {
+                return;
+            }
+            // if we didn't return here `msgpack_packer_write_u64` will raise a RangeError
+        }
+
         msgpack_packer_write_u64(pk, rb_big2ull(v));
     } else {
+        if(leading_zero_bits == 0) {
+            required_size += 1;
+        }
+
+        if(required_size > 8 && pk->has_bigint_ext_type) {
+            if(msgpack_packer_try_write_with_ext_type_lookup(pk, v)) {
+                return;
+            }
+            // if we didn't return here `msgpack_packer_write_u64` will raise a RangeError
+        }
+
         msgpack_packer_write_long_long(pk, rb_big2ll(v));
     }
 }

data/ext/msgpack/packer_class.c

@@ -28,6 +28,8 @@ VALUE cMessagePack_Packer;
 static ID s_to_msgpack;
 static ID s_write;
 
+static VALUE sym_compatibility_mode;
+
 //static VALUE s_packer_value;
 //static msgpack_packer_t* s_packer;
 
@@ -68,29 +70,28 @@ VALUE MessagePack_Packer_alloc(VALUE klass)
 
 VALUE MessagePack_Packer_initialize(int argc, VALUE* argv, VALUE self)
 {
+    if(argc > 2) {
+        rb_raise(rb_eArgError, "wrong number of arguments (%d for 0..2)", argc);
+    }
+
     VALUE io = Qnil;
     VALUE options = Qnil;
 
-    if(argc == 0 || (argc == 1 && argv[0] == Qnil)) {
-        /* Qnil */
-
-    } else if(argc == 1) {
-        VALUE v = argv[0];
-        if(rb_type(v) == T_HASH) {
-            options = v;
-        } else {
-            io = v;
-        }
-
-    } else if(argc == 2) {
+    if(argc >= 1) {
         io = argv[0];
+    }
+
+    if(argc == 2) {
         options = argv[1];
-        if(rb_type(options) != T_HASH) {
-            rb_raise(rb_eArgError, "expected Hash but found %s.", rb_obj_classname(options));
-        }
+    }
 
-    } else {
-        rb_raise(rb_eArgError, "wrong number of arguments (%d for 0..2)", argc);
+    if (options == Qnil && rb_type(io) == T_HASH) {
+        options = io;
+        io = Qnil;
+    }
+
+    if(options != Qnil) {
+        Check_Type(options, T_HASH);
     }
 
     PACKER(self, pk);
@@ -103,7 +104,7 @@ VALUE MessagePack_Packer_initialize(int argc, VALUE* argv, VALUE self)
     if(options != Qnil) {
         VALUE v;
 
-        v = rb_hash_aref(options, ID2SYM(rb_intern("compatibility_mode")));
+        v = rb_hash_aref(options, sym_compatibility_mode);
         msgpack_packer_set_compat(pk, RTEST(v));
     }
 
@@ -380,7 +381,7 @@ static VALUE Packer_register_type(int argc, VALUE* argv, VALUE self)
         rb_raise(rb_eArgError, "expected Module/Class but found %s.", rb_obj_classname(ext_module));
     }
 
-    msgpack_packer_ext_registry_put(&pk->ext_registry, ext_module, ext_type, proc, arg);
+    msgpack_packer_ext_registry_put(&pk->ext_registry, ext_module, ext_type, 0, proc, arg);
 
     if (ext_module == rb_cSymbol) {
         pk->has_symbol_ext_type = true;
@@ -412,6 +413,8 @@ void MessagePack_Packer_module_init(VALUE mMessagePack)
     s_to_msgpack = rb_intern("to_msgpack");
     s_write = rb_intern("write");
 
+    sym_compatibility_mode = ID2SYM(rb_intern("compatibility_mode"));
+
     msgpack_packer_static_init();
     msgpack_packer_ext_registry_static_init();
 

data/ext/msgpack/packer_ext_registry.c

@@ -43,12 +43,18 @@ void msgpack_packer_ext_registry_mark(msgpack_packer_ext_registry_t* pkrg)
 void msgpack_packer_ext_registry_dup(msgpack_packer_ext_registry_t* src,
         msgpack_packer_ext_registry_t* dst)
 {
-    dst->hash = RTEST(src->hash) ? rb_hash_dup(src->hash) : Qnil;
-    dst->cache = RTEST(src->cache) ? rb_hash_dup(src->cache): Qnil;
+    if(RTEST(src->hash) && !rb_obj_frozen_p(src->hash)) {
+        dst->hash = rb_hash_dup(src->hash);
+        dst->cache = RTEST(src->cache) ? rb_hash_dup(src->cache) : Qnil;
+    } else {
+        // If the type registry is frozen we can safely share it, and share the cache as well.
+        dst->hash = src->hash;
+        dst->cache = src->cache;
+    }
 }
 
 VALUE msgpack_packer_ext_registry_put(msgpack_packer_ext_registry_t* pkrg,
-        VALUE ext_module, int ext_type, VALUE proc, VALUE arg)
+        VALUE ext_module, int ext_type, int flags, VALUE proc, VALUE arg)
 {
     if (!RTEST(pkrg->hash)) {
         pkrg->hash = rb_hash_new();
@@ -58,5 +64,8 @@ VALUE msgpack_packer_ext_registry_put(msgpack_packer_ext_registry_t* pkrg,
         /* clear lookup cache not to miss added type */
         rb_hash_clear(pkrg->cache);
     }
-    return rb_hash_aset(pkrg->hash, ext_module, rb_ary_new3(3, INT2FIX(ext_type), proc, arg));
+
+    // TODO: Ruby embeded array limit is 3, merging `proc` and `arg` would be good.
+    VALUE entry = rb_ary_new3(4, INT2FIX(ext_type), proc, arg, INT2FIX(flags));
+    return rb_hash_aset(pkrg->hash, ext_module, entry);
 }

data/ext/msgpack/packer_ext_registry.h

@@ -21,6 +21,8 @@
 #include "compat.h"
 #include "ruby.h"
 
+#define MSGPACK_EXT_RECURSIVE 0b0001
+
 struct msgpack_packer_ext_registry_t;
 typedef struct msgpack_packer_ext_registry_t msgpack_packer_ext_registry_t;
 
@@ -44,7 +46,7 @@ void msgpack_packer_ext_registry_dup(msgpack_packer_ext_registry_t* src,
         msgpack_packer_ext_registry_t* dst);
 
 VALUE msgpack_packer_ext_registry_put(msgpack_packer_ext_registry_t* pkrg,
-        VALUE ext_module, int ext_type, VALUE proc, VALUE arg);
+        VALUE ext_module, int ext_type, int flags, VALUE proc, VALUE arg);
 
 static int msgpack_packer_ext_find_superclass(VALUE key, VALUE value, VALUE arg)
 {
@@ -60,12 +62,13 @@ static int msgpack_packer_ext_find_superclass(VALUE key, VALUE value, VALUE arg)
 }
 
 static inline VALUE msgpack_packer_ext_registry_fetch(msgpack_packer_ext_registry_t* pkrg,
-        VALUE lookup_class, int* ext_type_result)
+        VALUE lookup_class, int* ext_type_result, int* ext_flags_result)
 {
     // fetch lookup_class from hash, which is a hash to register classes
     VALUE type = rb_hash_lookup(pkrg->hash, lookup_class);
     if(type != Qnil) {
         *ext_type_result = FIX2INT(rb_ary_entry(type, 0));
+        *ext_flags_result = FIX2INT(rb_ary_entry(type, 3));
         return rb_ary_entry(type, 1);
     }
 
@@ -74,6 +77,7 @@ static inline VALUE msgpack_packer_ext_registry_fetch(msgpack_packer_ext_registr
         VALUE type_inht = rb_hash_lookup(pkrg->cache, lookup_class);
         if(type_inht != Qnil) {
             *ext_type_result = FIX2INT(rb_ary_entry(type_inht, 0));
+            *ext_flags_result = FIX2INT(rb_ary_entry(type_inht, 3));
             return rb_ary_entry(type_inht, 1);
         }
     }
@@ -82,7 +86,7 @@ static inline VALUE msgpack_packer_ext_registry_fetch(msgpack_packer_ext_registr
 }
 
 static inline VALUE msgpack_packer_ext_registry_lookup(msgpack_packer_ext_registry_t* pkrg,
-        VALUE instance, int* ext_type_result)
+        VALUE instance, int* ext_type_result, int* ext_flags_result)
 {
     VALUE type;
 
@@ -97,7 +101,7 @@ static inline VALUE msgpack_packer_ext_registry_lookup(msgpack_packer_ext_regist
     * `rb_class_of` returns the singleton_class if the object has one, or the "real class" otherwise.
     */
     VALUE lookup_class = rb_class_of(instance);
-    type = msgpack_packer_ext_registry_fetch(pkrg, lookup_class, ext_type_result);
+    type = msgpack_packer_ext_registry_fetch(pkrg, lookup_class, ext_type_result, ext_flags_result);
     if(type != Qnil) {
         return type;
     }
@@ -108,7 +112,7 @@ static inline VALUE msgpack_packer_ext_registry_lookup(msgpack_packer_ext_regist
      */
     VALUE real_class = rb_obj_class(instance);
     if(lookup_class != real_class) {
-        type = msgpack_packer_ext_registry_fetch(pkrg, real_class, ext_type_result);
+        type = msgpack_packer_ext_registry_fetch(pkrg, real_class, ext_type_result, ext_flags_result);
         if(type != Qnil) {
             return type;
         }
@@ -130,6 +134,7 @@ static inline VALUE msgpack_packer_ext_registry_lookup(msgpack_packer_ext_regist
         }
         rb_hash_aset(pkrg->cache, lookup_class, superclass_type);
         *ext_type_result = FIX2INT(rb_ary_entry(superclass_type, 0));
+        *ext_flags_result = FIX2INT(rb_ary_entry(superclass_type, 3));
         return rb_ary_entry(superclass_type, 1);
     }
 

data/ext/msgpack/unpacker.c

@@ -166,14 +166,17 @@ static inline int object_complete_ext(msgpack_unpacker_t* uk, int ext_type, VALU
         return object_complete_symbol(uk, rb_str_intern(str));
     }
 
-    VALUE proc = msgpack_unpacker_ext_registry_lookup(uk->ext_registry, ext_type);
+    int ext_flags;
+    VALUE proc = msgpack_unpacker_ext_registry_lookup(uk->ext_registry, ext_type, &ext_flags);
+
     if(proc != Qnil) {
-        VALUE obj = rb_funcall(proc, s_call, 1, str);
+        VALUE obj;
+        obj = rb_funcall(proc, s_call, 1, str == Qnil ? rb_str_buf_new(0) : str);
         return object_complete(uk, obj);
     }
 
     if(uk->allow_unknown_ext) {
-        VALUE obj = MessagePack_ExtensionValue_new(ext_type, str);
+        VALUE obj = MessagePack_ExtensionValue_new(ext_type, str == Qnil ? rb_str_buf_new(0) : str);
         return object_complete(uk, obj);
     }
 
@@ -283,6 +286,23 @@ static inline int read_raw_body_begin(msgpack_unpacker_t* uk, int raw_type)
 {
     /* assuming uk->reading_raw == Qnil */
 
+    int ext_flags;
+    VALUE proc;
+
+    if(!(raw_type == RAW_TYPE_STRING || raw_type == RAW_TYPE_BINARY)) {
+        proc = msgpack_unpacker_ext_registry_lookup(uk->ext_registry, raw_type, &ext_flags);
+        if(proc != Qnil && ext_flags & MSGPACK_EXT_RECURSIVE) {
+            VALUE obj;
+            uk->last_object = Qnil;
+            reset_head_byte(uk);
+            size_t ext_size = uk->reading_raw_remaining;
+            uk->reading_raw_remaining = 0;
+            obj = rb_funcall(proc, s_call, 1, uk->buffer.owner);
+            msgpack_buffer_skip(UNPACKER_BUFFER_(uk), ext_size);
+            return object_complete(uk, obj);
+        }
+    }
+
     /* try optimized read */
     size_t length = uk->reading_raw_remaining;
     if(length <= msgpack_buffer_top_readable_size(UNPACKER_BUFFER_(uk))) {
@@ -371,7 +391,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 uint8_t length = cb->u8;
                 int ext_type = (signed char) cb->buffer[1];
                 if(length == 0) {
-                    return object_complete_ext(uk, ext_type, rb_str_buf_new(0));
+                    return object_complete_ext(uk, ext_type, Qnil);
                 }
                 uk->reading_raw_remaining = length;
                 return read_raw_body_begin(uk, ext_type);
@@ -383,7 +403,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 uint16_t length = _msgpack_be16(cb->u16);
                 int ext_type = (signed char) cb->buffer[2];
                 if(length == 0) {
-                    return object_complete_ext(uk, ext_type, rb_str_buf_new(0));
+                    return object_complete_ext(uk, ext_type, Qnil);
                 }
                 uk->reading_raw_remaining = length;
                 return read_raw_body_begin(uk, ext_type);
@@ -395,7 +415,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 uint32_t length = _msgpack_be32(cb->u32);
                 int ext_type = (signed char) cb->buffer[4];
                 if(length == 0) {
-                    return object_complete_ext(uk, ext_type, rb_str_buf_new(0));
+                    return object_complete_ext(uk, ext_type, Qnil);
                 }
                 uk->reading_raw_remaining = length;
                 return read_raw_body_begin(uk, ext_type);