msgpack 1.4.5 → 1.5.2

data/ext/msgpack/unpacker.c

@@ -52,6 +52,16 @@ void msgpack_unpacker_static_destroy()
 
 #define HEAD_BYTE_REQUIRED 0xc1
 
+static inline msgpack_unpacker_stack_t* _msgpack_unpacker_new_stack(void) {
+#ifdef UNPACKER_STACK_RMEM
+    return msgpack_rmem_alloc(&s_stack_rmem);
+    /*memset(uk->stack, 0, MSGPACK_UNPACKER_STACK_CAPACITY);*/
+#else
+    /*uk->stack = calloc(MSGPACK_UNPACKER_STACK_CAPACITY, sizeof(msgpack_unpacker_stack_t));*/
+    return xmalloc(MSGPACK_UNPACKER_STACK_CAPACITY * sizeof(msgpack_unpacker_stack_t));
+#endif
+}
+
 msgpack_unpacker_t* _msgpack_unpacker_new(void)
 {
     msgpack_unpacker_t* uk = ZALLOC_N(msgpack_unpacker_t, 1);
@@ -63,26 +73,23 @@ msgpack_unpacker_t* _msgpack_unpacker_new(void)
     uk->last_object = Qnil;
     uk->reading_raw = Qnil;
 
-#ifdef UNPACKER_STACK_RMEM
-    uk->stack = msgpack_rmem_alloc(&s_stack_rmem);
-    /*memset(uk->stack, 0, MSGPACK_UNPACKER_STACK_CAPACITY);*/
-#else
-    /*uk->stack = calloc(MSGPACK_UNPACKER_STACK_CAPACITY, sizeof(msgpack_unpacker_stack_t));*/
-    uk->stack = xmalloc(MSGPACK_UNPACKER_STACK_CAPACITY * sizeof(msgpack_unpacker_stack_t));
-#endif
+    uk->stack = _msgpack_unpacker_new_stack();
     uk->stack_capacity = MSGPACK_UNPACKER_STACK_CAPACITY;
 
     return uk;
 }
 
+static inline void _msgpack_unpacker_free_stack(msgpack_unpacker_stack_t* stack) {
+    #ifdef UNPACKER_STACK_RMEM
+        msgpack_rmem_free(&s_stack_rmem, stack);
+    #else
+        xfree(stack);
+    #endif
+}
+
 void _msgpack_unpacker_destroy(msgpack_unpacker_t* uk)
 {
-#ifdef UNPACKER_STACK_RMEM
-    msgpack_rmem_free(&s_stack_rmem, uk->stack);
-#else
-    xfree(uk->stack);
-#endif
-
+    _msgpack_unpacker_free_stack(uk->stack);
     msgpack_buffer_destroy(UNPACKER_BUFFER_(uk));
 }
 
@@ -166,14 +173,17 @@ static inline int object_complete_ext(msgpack_unpacker_t* uk, int ext_type, VALU
         return object_complete_symbol(uk, rb_str_intern(str));
     }
 
-    VALUE proc = msgpack_unpacker_ext_registry_lookup(uk->ext_registry, ext_type);
+    int ext_flags;
+    VALUE proc = msgpack_unpacker_ext_registry_lookup(uk->ext_registry, ext_type, &ext_flags);
+
     if(proc != Qnil) {
-        VALUE obj = rb_funcall(proc, s_call, 1, str);
+        VALUE obj;
+        obj = rb_funcall(proc, s_call, 1, str == Qnil ? rb_str_buf_new(0) : str);
         return object_complete(uk, obj);
     }
 
     if(uk->allow_unknown_ext) {
-        VALUE obj = MessagePack_ExtensionValue_new(ext_type, str);
+        VALUE obj = MessagePack_ExtensionValue_new(ext_type, str == Qnil ? rb_str_buf_new(0) : str);
         return object_complete(uk, obj);
     }
 
@@ -283,6 +293,36 @@ static inline int read_raw_body_begin(msgpack_unpacker_t* uk, int raw_type)
 {
     /* assuming uk->reading_raw == Qnil */
 
+    int ext_flags;
+    VALUE proc;
+
+    if(!(raw_type == RAW_TYPE_STRING || raw_type == RAW_TYPE_BINARY)) {
+        proc = msgpack_unpacker_ext_registry_lookup(uk->ext_registry, raw_type, &ext_flags);
+        if(proc != Qnil && ext_flags & MSGPACK_EXT_RECURSIVE) {
+            VALUE obj;
+            uk->last_object = Qnil;
+            reset_head_byte(uk);
+            uk->reading_raw_remaining = 0;
+
+            msgpack_unpacker_stack_t* stack = uk->stack;
+            size_t stack_depth = uk->stack_depth;
+            size_t stack_capacity = uk->stack_capacity;
+
+            uk->stack = _msgpack_unpacker_new_stack();
+            uk->stack_depth = 0;
+            uk->stack_capacity = MSGPACK_UNPACKER_STACK_CAPACITY;
+
+            obj = rb_funcall(proc, s_call, 1, uk->buffer.owner);
+
+            _msgpack_unpacker_free_stack(uk->stack);
+            uk->stack = stack;
+            uk->stack_depth = stack_depth;
+            uk->stack_capacity = stack_capacity;
+
+            return object_complete(uk, obj);
+        }
+    }
+
     /* try optimized read */
     size_t length = uk->reading_raw_remaining;
     if(length <= msgpack_buffer_top_readable_size(UNPACKER_BUFFER_(uk))) {
@@ -371,7 +411,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 uint8_t length = cb->u8;
                 int ext_type = (signed char) cb->buffer[1];
                 if(length == 0) {
-                    return object_complete_ext(uk, ext_type, rb_str_buf_new(0));
+                    return object_complete_ext(uk, ext_type, Qnil);
                 }
                 uk->reading_raw_remaining = length;
                 return read_raw_body_begin(uk, ext_type);
@@ -383,7 +423,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 uint16_t length = _msgpack_be16(cb->u16);
                 int ext_type = (signed char) cb->buffer[2];
                 if(length == 0) {
-                    return object_complete_ext(uk, ext_type, rb_str_buf_new(0));
+                    return object_complete_ext(uk, ext_type, Qnil);
                 }
                 uk->reading_raw_remaining = length;
                 return read_raw_body_begin(uk, ext_type);
@@ -395,7 +435,7 @@ static int read_primitive(msgpack_unpacker_t* uk)
                 uint32_t length = _msgpack_be32(cb->u32);
                 int ext_type = (signed char) cb->buffer[4];
                 if(length == 0) {
-                    return object_complete_ext(uk, ext_type, rb_str_buf_new(0));
+                    return object_complete_ext(uk, ext_type, Qnil);
                 }
                 uk->reading_raw_remaining = length;
                 return read_raw_body_begin(uk, ext_type);

data/ext/msgpack/unpacker_class.c

@@ -33,6 +33,10 @@ static VALUE eUnexpectedTypeError;
 static VALUE eUnknownExtTypeError;
 static VALUE mTypeError;  // obsoleted. only for backward compatibility. See #86.
 
+static VALUE sym_symbolize_keys;
+static VALUE sym_freeze;
+static VALUE sym_allow_unknown_ext;
+
 #define UNPACKER(from, name) \
     msgpack_unpacker_t *name = NULL; \
     Data_Get_Struct(from, msgpack_unpacker_t, name); \
@@ -83,7 +87,7 @@ VALUE MessagePack_Unpacker_initialize(int argc, VALUE* argv, VALUE self)
     } else if(argc == 2) {
         io = argv[0];
         options = argv[1];
-        if(rb_type(options) != T_HASH) {
+        if(options != Qnil && rb_type(options) != T_HASH) {
             rb_raise(rb_eArgError, "expected Hash but found %s.", rb_obj_classname(options));
         }
 
@@ -100,13 +104,13 @@ VALUE MessagePack_Unpacker_initialize(int argc, VALUE* argv, VALUE self)
     if(options != Qnil) {
         VALUE v;
 
-        v = rb_hash_aref(options, ID2SYM(rb_intern("symbolize_keys")));
+        v = rb_hash_aref(options, sym_symbolize_keys);
         msgpack_unpacker_set_symbolized_keys(uk, RTEST(v));
 
-        v = rb_hash_aref(options, ID2SYM(rb_intern("freeze")));
+        v = rb_hash_aref(options, sym_freeze);
         msgpack_unpacker_set_freeze(uk, RTEST(v));
 
-        v = rb_hash_aref(options, ID2SYM(rb_intern("allow_unknown_ext")));
+        v = rb_hash_aref(options, sym_allow_unknown_ext);
         msgpack_unpacker_set_allow_unknown_ext(uk, RTEST(v));
     }
 
@@ -143,6 +147,7 @@ NORETURN(static void raise_unpacker_error(int r))
     case PRIMITIVE_UNEXPECTED_TYPE:
         rb_raise(eUnexpectedTypeError, "unexpected type");
     case PRIMITIVE_UNEXPECTED_EXT_TYPE:
+    // rb_bug("unexpected extension type");
         rb_raise(eUnknownExtTypeError, "unexpected extension type");
     default:
         rb_raise(eUnpackError, "logically unknown error %d", r);
@@ -361,7 +366,7 @@ static VALUE Unpacker_register_type(int argc, VALUE* argv, VALUE self)
         rb_raise(rb_eRangeError, "integer %d too big to convert to `signed char'", ext_type);
     }
 
-    msgpack_unpacker_ext_registry_put(&uk->ext_registry, ext_module, ext_type, proc, arg);
+    msgpack_unpacker_ext_registry_put(&uk->ext_registry, ext_module, ext_type, 0, proc, arg);
 
     return Qnil;
 }
@@ -411,6 +416,10 @@ void MessagePack_Unpacker_module_init(VALUE mMessagePack)
 
     eUnknownExtTypeError = rb_define_class_under(mMessagePack, "UnknownExtTypeError", eUnpackError);
 
+    sym_symbolize_keys = ID2SYM(rb_intern("symbolize_keys"));
+    sym_freeze = ID2SYM(rb_intern("freeze"));
+    sym_allow_unknown_ext = ID2SYM(rb_intern("allow_unknown_ext"));
+
     rb_define_alloc_func(cMessagePack_Unpacker, MessagePack_Unpacker_alloc);
 
     rb_define_method(cMessagePack_Unpacker, "initialize", MessagePack_Unpacker_initialize, -1);

data/ext/msgpack/unpacker_ext_registry.c

@@ -77,9 +77,10 @@ void msgpack_unpacker_ext_registry_release(msgpack_unpacker_ext_registry_t* ukrg
 }
 
 void msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t** ukrg,
-        VALUE ext_module, int ext_type, VALUE proc, VALUE arg)
+        VALUE ext_module, int ext_type, int flags, VALUE proc, VALUE arg)
 {
     msgpack_unpacker_ext_registry_t* ext_registry = msgpack_unpacker_ext_registry_cow(*ukrg);
-    ext_registry->array[ext_type + 128] = rb_ary_new3(3, ext_module, proc, arg);
+
+    ext_registry->array[ext_type + 128] = rb_ary_new3(4, ext_module, proc, arg, INT2FIX(flags));
     *ukrg = ext_registry;
 }

data/ext/msgpack/unpacker_ext_registry.h

@@ -21,6 +21,8 @@
 #include "compat.h"
 #include "ruby.h"
 
+#define MSGPACK_EXT_RECURSIVE 0b0001
+
 struct msgpack_unpacker_ext_registry_t;
 typedef struct msgpack_unpacker_ext_registry_t msgpack_unpacker_ext_registry_t;
 
@@ -46,14 +48,15 @@ static inline void msgpack_unpacker_ext_registry_borrow(msgpack_unpacker_ext_reg
 void msgpack_unpacker_ext_registry_mark(msgpack_unpacker_ext_registry_t* ukrg);
 
 void msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t** ukrg,
-        VALUE ext_module, int ext_type, VALUE proc, VALUE arg);
+        VALUE ext_module, int ext_type, int flags, VALUE proc, VALUE arg);
 
 static inline VALUE msgpack_unpacker_ext_registry_lookup(msgpack_unpacker_ext_registry_t* ukrg,
-        int ext_type)
+        int ext_type, int* ext_flags_result)
 {
     if (ukrg) {
         VALUE entry = ukrg->array[ext_type + 128];
         if (entry != Qnil) {
+            *ext_flags_result = FIX2INT(rb_ary_entry(entry, 3));
             return rb_ary_entry(entry, 1);
         }
     }

data/lib/msgpack/bigint.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module MessagePack
+  module Bigint
+    # We split the bigint in 32bits chunks so that individual part fits into
+    # a MRI immediate Integer.
+    CHUNK_BITLENGTH = 32
+    FORMAT = 'CL>*'
+
+    if Integer.instance_method(:[]).arity != 1 # Ruby 2.7 and newer
+      # Starting from Ruby 2.7 we can address arbitrary bitranges inside an Integer with Integer#[]
+      # This allows to not allocate any Integer.
+      def self.to_msgpack_ext(bigint)
+        members = []
+
+        if bigint < 0
+          bigint = -bigint
+          members << 1
+        else
+          members << 0
+        end
+
+        offset = 0
+        length = bigint.bit_length
+        while offset < length
+          members << bigint[offset, CHUNK_BITLENGTH]
+          offset += CHUNK_BITLENGTH
+        end
+
+        members.pack(FORMAT)
+      end
+    else
+      # On 2.6 and older since we can't address arbitrary bitranges, so we fallback to shifting the bigint.
+      # This means that after each shift, we may allocate another Integer instance.
+      BASE = (2**CHUNK_BITLENGTH) - 1
+      def self.to_msgpack_ext(bigint)
+        members = []
+
+        if bigint < 0
+          bigint = -bigint
+          members << 1
+        else
+          members << 0
+        end
+
+        while bigint > 0
+          members << (bigint & BASE)
+          bigint = bigint >> CHUNK_BITLENGTH
+        end
+
+        members.pack(FORMAT)
+      end
+    end
+
+    def self.from_msgpack_ext(data)
+      parts = data.unpack(FORMAT)
+
+      sign = parts.shift
+      sum = parts.pop.to_i
+
+      parts.reverse_each do |part|
+        sum = sum << CHUNK_BITLENGTH
+        sum += part
+      end
+
+      sign == 0 ? sum : -sum
+    end
+  end
+end

data/lib/msgpack/factory.rb

@@ -77,5 +77,108 @@ module MessagePack
       packer.full_pack
     end
     alias :pack :dump
+
+    def pool(size = 1, **options)
+      Pool.new(
+        frozen? ? self : dup.freeze,
+        size,
+        options.empty? ? nil : options,
+      )
+    end
+
+    class Pool
+      if RUBY_ENGINE == "ruby"
+        class AbstractPool
+          def initialize(size, &block)
+            @size = size
+            @new_member = block
+            @members = []
+          end
+
+          def checkout
+            @members.pop || @new_member.call
+          end
+
+          def checkin(member)
+            # If the pool is already full, we simply drop the extra member.
+            # This is because contrary to a connection pool, creating an extra instance
+            # is extremely unlikely to cause some kind of resource exhaustion.
+            #
+            # We could cycle the members (keep the newer one) but first It's more work and second
+            # the older member might have been created pre-fork, so it might be at least partially
+            # in shared memory.
+            if member && @members.size < @size
+              member.reset
+              @members << member
+            end
+          end
+        end
+      else
+        class AbstractPool
+          def initialize(size, &block)
+            @size = size
+            @new_member = block
+            @members = []
+            @mutex = Mutex.new
+          end
+
+          def checkout
+            @mutex.synchronize { @members.pop } || @new_member.call
+          end
+
+          def checkin(member)
+            @mutex.synchronize do
+              if member && @members.size < @size
+                member.reset
+                @members << member
+              end
+            end
+          end
+        end
+      end
+
+      class PackerPool < AbstractPool
+        private
+
+        def reset(packer)
+          packer.clear
+        end
+      end
+
+      class UnpackerPool < AbstractPool
+        private
+
+        def reset(unpacker)
+          unpacker.reset
+        end
+      end
+
+      def initialize(factory, size, options = nil)
+        options = nil if !options || options.empty?
+        @factory = factory
+        @packers = PackerPool.new(size) { factory.packer(options) }
+        @unpackers = UnpackerPool.new(size) { factory.unpacker(options) }
+      end
+
+      def load(data)
+        unpacker = @unpackers.checkout
+        begin
+          unpacker.feed_reference(data)
+          unpacker.full_unpack
+        ensure
+          @unpackers.checkin(unpacker)
+        end
+      end
+
+      def dump(object)
+        packer = @packers.checkout
+        begin
+          packer.write(object)
+          packer.full_pack
+        ensure
+          @packers.checkin(packer)
+        end
+      end
+    end
   end
 end

data/lib/msgpack/version.rb

@@ -1,5 +1,5 @@
 module MessagePack
-  VERSION = "1.4.5"
+  VERSION = "1.5.2"
   # Note for maintainers:
   #  Don't miss building/releasing the JRuby version (rake buld:java)
   #  See "How to build -java rubygems" in README for more details.

data/lib/msgpack.rb

@@ -17,16 +17,15 @@ require "msgpack/time"
 
 module MessagePack
   DefaultFactory = MessagePack::Factory.new
-  DEFAULT_EMPTY_PARAMS = {}.freeze
 
   def load(src, param = nil)
     unpacker = nil
 
     if src.is_a? String
-      unpacker = DefaultFactory.unpacker param || DEFAULT_EMPTY_PARAMS
+      unpacker = DefaultFactory.unpacker param
       unpacker.feed_reference src
     else
-      unpacker = DefaultFactory.unpacker src, param || DEFAULT_EMPTY_PARAMS
+      unpacker = DefaultFactory.unpacker src, param
     end
 
     unpacker.full_unpack
@@ -36,8 +35,8 @@ module MessagePack
   module_function :load
   module_function :unpack
 
-  def pack(v, *rest)
-    packer = DefaultFactory.packer(*rest)
+  def pack(v, io = nil, options = nil)
+    packer = DefaultFactory.packer(io, options)
     packer.write v
     packer.full_pack
   end

data/spec/bigint_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe MessagePack::Bigint do
+  it 'serialize and deserialize arbitrary sized integer' do
+    [
+      1,
+      -1,
+      120938120391283122132313,
+      -21903120391203912391023920332103,
+      210290021321301203912933021323,
+    ].each do |int|
+      expect(MessagePack::Bigint.from_msgpack_ext(MessagePack::Bigint.to_msgpack_ext(int))).to be == int
+    end
+  end
+
+  it 'has a stable format' do
+    {
+      120938120391283122132313 => "\x00\x9F\xF4UY\x11\x92\x9A?\x00\x00\x19\x9C".b,
+      -21903120391203912391023920332103 => "\x01/\xB2\xBDG\xBD\xDE\xAA\xEBt\xCC\x8A\xC1\x00\x00\x01\x14".b,
+      210290021321301203912933021323 => "\x00\xC4\xD8\x96\x8Bm\xCB\xC7\x03\xA7{\xD4\"\x00\x00\x00\x02".b,
+    }.each do |int, payload|
+      expect(MessagePack::Bigint.to_msgpack_ext(int)).to be == payload
+      expect(MessagePack::Bigint.from_msgpack_ext(payload)).to be == int
+    end
+  end
+end