msgpack 1.4.4 → 1.5.1

data/ext/msgpack/unpacker_ext_registry.c

@@ -27,36 +27,60 @@ void msgpack_unpacker_ext_registry_static_init()
     s_dup = rb_intern("dup");
 }
 
+
 void msgpack_unpacker_ext_registry_static_destroy()
 { }
 
-void msgpack_unpacker_ext_registry_init(msgpack_unpacker_ext_registry_t* ukrg)
+void msgpack_unpacker_ext_registry_mark(msgpack_unpacker_ext_registry_t* ukrg)
 {
-    for(int i=0; i < 256; i++) {
-        ukrg->array[i] = Qnil;
+    if (ukrg) {
+        for(int i=0; i < 256; i++) {
+            if (ukrg->array[i] != Qnil) {
+                rb_gc_mark(ukrg->array[i]);
+            }
+        }
     }
 }
 
-void msgpack_unpacker_ext_registry_mark(msgpack_unpacker_ext_registry_t* ukrg)
+msgpack_unpacker_ext_registry_t* msgpack_unpacker_ext_registry_cow(msgpack_unpacker_ext_registry_t* src)
 {
-    for(int i=0; i < 256; i++) {
-        rb_gc_mark(ukrg->array[i]);
+    msgpack_unpacker_ext_registry_t* dst;
+    if (src) {
+        if (src->borrow_count) {
+            dst = ALLOC(msgpack_unpacker_ext_registry_t);
+            dst->borrow_count = 0;
+            MEMCPY(dst->array, src->array, VALUE, 256);
+            msgpack_unpacker_ext_registry_release(src);
+            return dst;
+        } else {
+            return src;
+        }
+    } else {
+        dst = ALLOC(msgpack_unpacker_ext_registry_t);
+        dst->borrow_count = 0;
+        for(int i=0; i < 256; i++) {
+            dst->array[i] = Qnil;
+        }
+        return dst;
     }
 }
 
-void msgpack_unpacker_ext_registry_dup(msgpack_unpacker_ext_registry_t* src,
-        msgpack_unpacker_ext_registry_t* dst)
+void msgpack_unpacker_ext_registry_release(msgpack_unpacker_ext_registry_t* ukrg)
 {
-    for(int i=0; i < 256; i++) {
-        dst->array[i] = src->array[i];
+    if (ukrg) {
+        if (ukrg->borrow_count) {
+            ukrg->borrow_count--;
+        } else {
+            xfree(ukrg);
+        }
     }
 }
 
-VALUE msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t* ukrg,
-        VALUE ext_module, int ext_type, VALUE proc, VALUE arg)
+void msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t** ukrg,
+        VALUE ext_module, int ext_type, int flags, VALUE proc, VALUE arg)
 {
-    VALUE e = rb_ary_new3(3, ext_module, proc, arg);
-    VALUE before = ukrg->array[ext_type + 128];
-    ukrg->array[ext_type + 128] = e;
-    return before;
+    msgpack_unpacker_ext_registry_t* ext_registry = msgpack_unpacker_ext_registry_cow(*ukrg);
+
+    ext_registry->array[ext_type + 128] = rb_ary_new3(4, ext_module, proc, arg, INT2FIX(flags));
+    *ukrg = ext_registry;
 }

data/ext/msgpack/unpacker_ext_registry.h

@@ -21,39 +21,46 @@
 #include "compat.h"
 #include "ruby.h"
 
+#define MSGPACK_EXT_RECURSIVE 0b0001
+
 struct msgpack_unpacker_ext_registry_t;
 typedef struct msgpack_unpacker_ext_registry_t msgpack_unpacker_ext_registry_t;
 
 struct msgpack_unpacker_ext_registry_t {
+    unsigned int borrow_count;
     VALUE array[256];
-    //int bitmap;
 };
 
 void msgpack_unpacker_ext_registry_static_init();
 
 void msgpack_unpacker_ext_registry_static_destroy();
 
-void msgpack_unpacker_ext_registry_init(msgpack_unpacker_ext_registry_t* ukrg);
+void msgpack_unpacker_ext_registry_release(msgpack_unpacker_ext_registry_t* ukrg);
 
-static inline void msgpack_unpacker_ext_registry_destroy(msgpack_unpacker_ext_registry_t* ukrg)
-{ }
+static inline void msgpack_unpacker_ext_registry_borrow(msgpack_unpacker_ext_registry_t* src, msgpack_unpacker_ext_registry_t** dst)
+{
+    if (src) {
+        src->borrow_count++;
+        *dst = src;
+    }
+}
 
 void msgpack_unpacker_ext_registry_mark(msgpack_unpacker_ext_registry_t* ukrg);
 
-void msgpack_unpacker_ext_registry_dup(msgpack_unpacker_ext_registry_t* src,
-        msgpack_unpacker_ext_registry_t* dst);
-
-VALUE msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t* ukrg,
-        VALUE ext_module, int ext_type, VALUE proc, VALUE arg);
+void msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t** ukrg,
+        VALUE ext_module, int ext_type, int flags, VALUE proc, VALUE arg);
 
 static inline VALUE msgpack_unpacker_ext_registry_lookup(msgpack_unpacker_ext_registry_t* ukrg,
-        int ext_type)
+        int ext_type, int* ext_flags_result)
 {
-    VALUE e = ukrg->array[ext_type + 128];
-    if(e == Qnil) {
-        return Qnil;
+    if (ukrg) {
+        VALUE entry = ukrg->array[ext_type + 128];
+        if (entry != Qnil) {
+            *ext_flags_result = FIX2INT(rb_ary_entry(entry, 3));
+            return rb_ary_entry(entry, 1);
+        }
     }
-    return rb_ary_entry(e, 1);
+    return Qnil;
 }
 
 #endif

data/lib/msgpack/bigint.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module MessagePack
+  module Bigint
+    # We split the bigint in 32bits chunks so that individual part fits into
+    # a MRI immediate Integer.
+    CHUNK_BITLENGTH = 32
+    FORMAT = 'CL>*'
+
+    if Integer.instance_method(:[]).arity != 1 # Ruby 2.7 and newer
+      # Starting from Ruby 2.7 we can address arbitrary bitranges inside an Integer with Integer#[]
+      # This allows to not allocate any Integer.
+      def self.to_msgpack_ext(bigint)
+        members = []
+
+        if bigint < 0
+          bigint = -bigint
+          members << 1
+        else
+          members << 0
+        end
+
+        offset = 0
+        length = bigint.bit_length
+        while offset < length
+          members << bigint[offset, CHUNK_BITLENGTH]
+          offset += CHUNK_BITLENGTH
+        end
+
+        members.pack(FORMAT)
+      end
+    else
+      # On 2.6 and older since we can't address arbitrary bitranges, so we fallback to shifting the bigint.
+      # This means that after each shift, we may allocate another Integer instance.
+      BASE = (2**CHUNK_BITLENGTH) - 1
+      def self.to_msgpack_ext(bigint)
+        members = []
+
+        if bigint < 0
+          bigint = -bigint
+          members << 1
+        else
+          members << 0
+        end
+
+        while bigint > 0
+          members << (bigint & BASE)
+          bigint = bigint >> CHUNK_BITLENGTH
+        end
+
+        members.pack(FORMAT)
+      end
+    end
+
+    def self.from_msgpack_ext(data)
+      parts = data.unpack(FORMAT)
+
+      sign = parts.shift
+      sum = parts.pop.to_i
+
+      parts.reverse_each do |part|
+        sum = sum << CHUNK_BITLENGTH
+        sum += part
+      end
+
+      sign == 0 ? sum : -sum
+    end
+  end
+end

data/lib/msgpack/factory.rb

@@ -77,5 +77,108 @@ module MessagePack
       packer.full_pack
     end
     alias :pack :dump
+
+    def pool(size = 1, **options)
+      Pool.new(
+        frozen? ? self : dup.freeze,
+        size,
+        options.empty? ? nil : options,
+      )
+    end
+
+    class Pool
+      if RUBY_ENGINE == "ruby"
+        class AbstractPool
+          def initialize(size, &block)
+            @size = size
+            @new_member = block
+            @members = []
+          end
+
+          def checkout
+            @members.pop || @new_member.call
+          end
+
+          def checkin(member)
+            # If the pool is already full, we simply drop the extra member.
+            # This is because contrary to a connection pool, creating an extra instance
+            # is extremely unlikely to cause some kind of resource exhaustion.
+            #
+            # We could cycle the members (keep the newer one) but first It's more work and second
+            # the older member might have been created pre-fork, so it might be at least partially
+            # in shared memory.
+            if member && @members.size < @size
+              member.reset
+              @members << member
+            end
+          end
+        end
+      else
+        class AbstractPool
+          def initialize(size, &block)
+            @size = size
+            @new_member = block
+            @members = []
+            @mutex = Mutex.new
+          end
+
+          def checkout
+            @mutex.synchronize { @members.pop } || @new_member.call
+          end
+
+          def checkin(member)
+            @mutex.synchronize do
+              if member && @members.size < @size
+                member.reset
+                @members << member
+              end
+            end
+          end
+        end
+      end
+
+      class PackerPool < AbstractPool
+        private
+
+        def reset(packer)
+          packer.clear
+        end
+      end
+
+      class UnpackerPool < AbstractPool
+        private
+
+        def reset(unpacker)
+          unpacker.reset
+        end
+      end
+
+      def initialize(factory, size, options = nil)
+        options = nil if !options || options.empty?
+        @factory = factory
+        @packers = PackerPool.new(size) { factory.packer(options) }
+        @unpackers = UnpackerPool.new(size) { factory.unpacker(options) }
+      end
+
+      def load(data)
+        unpacker = @unpackers.checkout
+        begin
+          unpacker.feed_reference(data)
+          unpacker.full_unpack
+        ensure
+          @unpackers.checkin(unpacker)
+        end
+      end
+
+      def dump(object)
+        packer = @packers.checkout
+        begin
+          packer.write(object)
+          packer.full_pack
+        ensure
+          @packers.checkin(packer)
+        end
+      end
+    end
   end
 end

data/lib/msgpack/symbol.rb

@@ -14,6 +14,13 @@ class Symbol
     # The canonical way to do it for symbols would be:
     #  data.unpack1('A*').to_sym
     # However in this instance we can take a shortcut
-    data.to_sym
+
+    # We assume the string encoding is UTF-8, and let Ruby create either
+    # an ASCII symbol or UTF-8 symbol.
+    data.force_encoding(Encoding::UTF_8).to_sym
+  rescue EncodingError
+    # If somehow the string wasn't valid UTF-8 not valid ASCII, we fallback
+    # to what has been the historical behavior of creating a binary symbol
+    data.force_encoding(Encoding::BINARY).to_sym
   end
 end

data/lib/msgpack/version.rb

@@ -1,5 +1,5 @@
 module MessagePack
-  VERSION = "1.4.4"
+  VERSION = "1.5.1"
   # Note for maintainers:
   #  Don't miss building/releasing the JRuby version (rake buld:java)
   #  See "How to build -java rubygems" in README for more details.

data/lib/msgpack.rb

@@ -17,16 +17,15 @@ require "msgpack/time"
 
 module MessagePack
   DefaultFactory = MessagePack::Factory.new
-  DEFAULT_EMPTY_PARAMS = {}.freeze
 
   def load(src, param = nil)
     unpacker = nil
 
     if src.is_a? String
-      unpacker = DefaultFactory.unpacker param || DEFAULT_EMPTY_PARAMS
+      unpacker = DefaultFactory.unpacker param
       unpacker.feed_reference src
     else
-      unpacker = DefaultFactory.unpacker src, param || DEFAULT_EMPTY_PARAMS
+      unpacker = DefaultFactory.unpacker src, param
     end
 
     unpacker.full_unpack
@@ -36,8 +35,8 @@ module MessagePack
   module_function :load
   module_function :unpack
 
-  def pack(v, *rest)
-    packer = DefaultFactory.packer(*rest)
+  def pack(v, io = nil, options = nil)
+    packer = DefaultFactory.packer(io, options)
     packer.write v
     packer.full_pack
   end

data/spec/bigint_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe MessagePack::Bigint do
+  it 'serialize and deserialize arbitrary sized integer' do
+    [
+      1,
+      -1,
+      120938120391283122132313,
+      -21903120391203912391023920332103,
+      210290021321301203912933021323,
+    ].each do |int|
+      expect(MessagePack::Bigint.from_msgpack_ext(MessagePack::Bigint.to_msgpack_ext(int))).to be == int
+    end
+  end
+
+  it 'has a stable format' do
+    {
+      120938120391283122132313 => "\x00\x9F\xF4UY\x11\x92\x9A?\x00\x00\x19\x9C".b,
+      -21903120391203912391023920332103 => "\x01/\xB2\xBDG\xBD\xDE\xAA\xEBt\xCC\x8A\xC1\x00\x00\x01\x14".b,
+      210290021321301203912933021323 => "\x00\xC4\xD8\x96\x8Bm\xCB\xC7\x03\xA7{\xD4\"\x00\x00\x00\x02".b,
+    }.each do |int, payload|
+      expect(MessagePack::Bigint.to_msgpack_ext(int)).to be == payload
+      expect(MessagePack::Bigint.from_msgpack_ext(payload)).to be == int
+    end
+  end
+end