msgpack 1.4.3 → 1.5.0

data/ext/msgpack/unpacker_ext_registry.h

@@ -21,39 +21,46 @@
 #include "compat.h"
 #include "ruby.h"
 
+#define MSGPACK_EXT_RECURSIVE 0b0001
+
 struct msgpack_unpacker_ext_registry_t;
 typedef struct msgpack_unpacker_ext_registry_t msgpack_unpacker_ext_registry_t;
 
 struct msgpack_unpacker_ext_registry_t {
+    unsigned int borrow_count;
     VALUE array[256];
-    //int bitmap;
 };
 
 void msgpack_unpacker_ext_registry_static_init();
 
 void msgpack_unpacker_ext_registry_static_destroy();
 
-void msgpack_unpacker_ext_registry_init(msgpack_unpacker_ext_registry_t* ukrg);
+void msgpack_unpacker_ext_registry_release(msgpack_unpacker_ext_registry_t* ukrg);
 
-static inline void msgpack_unpacker_ext_registry_destroy(msgpack_unpacker_ext_registry_t* ukrg)
-{ }
+static inline void msgpack_unpacker_ext_registry_borrow(msgpack_unpacker_ext_registry_t* src, msgpack_unpacker_ext_registry_t** dst)
+{
+    if (src) {
+        src->borrow_count++;
+        *dst = src;
+    }
+}
 
 void msgpack_unpacker_ext_registry_mark(msgpack_unpacker_ext_registry_t* ukrg);
 
-void msgpack_unpacker_ext_registry_dup(msgpack_unpacker_ext_registry_t* src,
-        msgpack_unpacker_ext_registry_t* dst);
-
-VALUE msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t* ukrg,
-        VALUE ext_module, int ext_type, VALUE proc, VALUE arg);
+void msgpack_unpacker_ext_registry_put(msgpack_unpacker_ext_registry_t** ukrg,
+        VALUE ext_module, int ext_type, int flags, VALUE proc, VALUE arg);
 
 static inline VALUE msgpack_unpacker_ext_registry_lookup(msgpack_unpacker_ext_registry_t* ukrg,
-        int ext_type)
+        int ext_type, int* ext_flags_result)
 {
-    VALUE e = ukrg->array[ext_type + 128];
-    if(e == Qnil) {
-        return Qnil;
+    if (ukrg) {
+        VALUE entry = ukrg->array[ext_type + 128];
+        if (entry != Qnil) {
+            *ext_flags_result = FIX2INT(rb_ary_entry(entry, 3));
+            return rb_ary_entry(entry, 1);
+        }
     }
-    return rb_ary_entry(e, 1);
+    return Qnil;
 }
 
 #endif

data/lib/msgpack/bigint.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module MessagePack
+  module Bigint
+    # We split the bigint in 32bits chunks so that individual part fits into
+    # a MRI immediate Integer.
+    CHUNK_BITLENGTH = 32
+    FORMAT = 'CL>*'
+
+    if Integer.instance_method(:[]).arity != 1 # Ruby 2.7 and newer
+      # Starting from Ruby 2.7 we can address arbitrary bitranges inside an Integer with Integer#[]
+      # This allows to not allocate any Integer.
+      def self.to_msgpack_ext(bigint)
+        members = []
+
+        if bigint < 0
+          bigint = -bigint
+          members << 1
+        else
+          members << 0
+        end
+
+        offset = 0
+        length = bigint.bit_length
+        while offset < length
+          members << bigint[offset, CHUNK_BITLENGTH]
+          offset += CHUNK_BITLENGTH
+        end
+
+        members.pack(FORMAT)
+      end
+    else
+      # On 2.6 and older since we can't address arbitrary bitranges, so we fallback to shifting the bigint.
+      # This means that after each shift, we may allocate another Integer instance.
+      BASE = (2**CHUNK_BITLENGTH) - 1
+      def self.to_msgpack_ext(bigint)
+        members = []
+
+        if bigint < 0
+          bigint = -bigint
+          members << 1
+        else
+          members << 0
+        end
+
+        while bigint > 0
+          members << (bigint & BASE)
+          bigint = bigint >> CHUNK_BITLENGTH
+        end
+
+        members.pack(FORMAT)
+      end
+    end
+
+    def self.from_msgpack_ext(data)
+      parts = data.unpack(FORMAT)
+
+      sign = parts.shift
+      sum = parts.pop.to_i
+
+      parts.reverse_each do |part|
+        sum = sum << CHUNK_BITLENGTH
+        sum += part
+      end
+
+      sign == 0 ? sum : -sum
+    end
+  end
+end

data/lib/msgpack/factory.rb

@@ -77,5 +77,108 @@ module MessagePack
       packer.full_pack
     end
     alias :pack :dump
+
+    def pool(size = 1, **options)
+      Pool.new(
+        frozen? ? self : dup.freeze,
+        size,
+        options.empty? ? nil : options,
+      )
+    end
+
+    class Pool
+      if RUBY_ENGINE == "ruby"
+        class AbstractPool
+          def initialize(size, &block)
+            @size = size
+            @new_member = block
+            @members = []
+          end
+
+          def checkout
+            @members.pop || @new_member.call
+          end
+
+          def checkin(member)
+            # If the pool is already full, we simply drop the extra member.
+            # This is because contrary to a connection pool, creating an extra instance
+            # is extremely unlikely to cause some kind of resource exhaustion.
+            #
+            # We could cycle the members (keep the newer one) but first It's more work and second
+            # the older member might have been created pre-fork, so it might be at least partially
+            # in shared memory.
+            if member && @members.size < @size
+              member.reset
+              @members << member
+            end
+          end
+        end
+      else
+        class AbstractPool
+          def initialize(size, &block)
+            @size = size
+            @new_member = block
+            @members = []
+            @mutex = Mutex.new
+          end
+
+          def checkout
+            @mutex.synchronize { @members.pop } || @new_member.call
+          end
+
+          def checkin(member)
+            @mutex.synchronize do
+              if member && @members.size < @size
+                member.reset
+                @members << member
+              end
+            end
+          end
+        end
+      end
+
+      class PackerPool < AbstractPool
+        private
+
+        def reset(packer)
+          packer.clear
+        end
+      end
+
+      class UnpackerPool < AbstractPool
+        private
+
+        def reset(unpacker)
+          unpacker.reset
+        end
+      end
+
+      def initialize(factory, size, options = nil)
+        options = nil if !options || options.empty?
+        @factory = factory
+        @packers = PackerPool.new(size) { factory.packer(options) }
+        @unpackers = UnpackerPool.new(size) { factory.unpacker(options) }
+      end
+
+      def load(data)
+        unpacker = @unpackers.checkout
+        begin
+          unpacker.feed_reference(data)
+          unpacker.full_unpack
+        ensure
+          @unpackers.checkin(unpacker)
+        end
+      end
+
+      def dump(object)
+        packer = @packers.checkout
+        begin
+          packer.write(object)
+          packer.full_pack
+        ensure
+          @packers.checkin(packer)
+        end
+      end
+    end
   end
 end

data/lib/msgpack/symbol.rb

@@ -14,6 +14,13 @@ class Symbol
     # The canonical way to do it for symbols would be:
     #  data.unpack1('A*').to_sym
     # However in this instance we can take a shortcut
-    data.to_sym
+
+    # We assume the string encoding is UTF-8, and let Ruby create either
+    # an ASCII symbol or UTF-8 symbol.
+    data.force_encoding(Encoding::UTF_8).to_sym
+  rescue EncodingError
+    # If somehow the string wasn't valid UTF-8 not valid ASCII, we fallback
+    # to what has been the historical behavior of creating a binary symbol
+    data.force_encoding(Encoding::BINARY).to_sym
   end
 end

data/lib/msgpack/version.rb

@@ -1,5 +1,5 @@
 module MessagePack
-  VERSION = "1.4.3"
+  VERSION = "1.5.0"
   # Note for maintainers:
   #  Don't miss building/releasing the JRuby version (rake buld:java)
   #  See "How to build -java rubygems" in README for more details.

data/lib/msgpack.rb

@@ -17,16 +17,15 @@ require "msgpack/time"
 
 module MessagePack
   DefaultFactory = MessagePack::Factory.new
-  DEFAULT_EMPTY_PARAMS = {}.freeze
 
   def load(src, param = nil)
     unpacker = nil
 
     if src.is_a? String
-      unpacker = DefaultFactory.unpacker param || DEFAULT_EMPTY_PARAMS
+      unpacker = DefaultFactory.unpacker param
       unpacker.feed_reference src
     else
-      unpacker = DefaultFactory.unpacker src, param || DEFAULT_EMPTY_PARAMS
+      unpacker = DefaultFactory.unpacker src, param
     end
 
     unpacker.full_unpack
@@ -36,8 +35,8 @@ module MessagePack
   module_function :load
   module_function :unpack
 
-  def pack(v, *rest)
-    packer = DefaultFactory.packer(*rest)
+  def pack(v, io = nil, options = nil)
+    packer = DefaultFactory.packer(io, options)
     packer.write v
     packer.full_pack
   end

data/msgpack.gemspec

@@ -18,13 +18,12 @@ Gem::Specification.new do |s|
     s.files = `git ls-files`.split("\n")
     s.extensions = ["ext/msgpack/extconf.rb"]
   end
-  s.test_files = `git ls-files -- {test,spec}/*`.split("\n")
 
   s.required_ruby_version = ">= 2.4"
 
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'rake'
-  s.add_development_dependency 'rake-compiler'
+  s.add_development_dependency 'rake-compiler', ['>= 1.1.9']
   s.add_development_dependency 'rspec', ['~> 3.3']
   s.add_development_dependency 'yard'
   s.add_development_dependency 'json'

data/spec/bigint_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe MessagePack::Bigint do
+  it 'serialize and deserialize arbitrary sized integer' do
+    [
+      1,
+      -1,
+      120938120391283122132313,
+      -21903120391203912391023920332103,
+      210290021321301203912933021323,
+    ].each do |int|
+      expect(MessagePack::Bigint.from_msgpack_ext(MessagePack::Bigint.to_msgpack_ext(int))).to be == int
+    end
+  end
+
+  it 'has a stable format' do
+    {
+      120938120391283122132313 => "\x00\x9F\xF4UY\x11\x92\x9A?\x00\x00\x19\x9C".b,
+      -21903120391203912391023920332103 => "\x01/\xB2\xBDG\xBD\xDE\xAA\xEBt\xCC\x8A\xC1\x00\x00\x01\x14".b,
+      210290021321301203912933021323 => "\x00\xC4\xD8\x96\x8Bm\xCB\xC7\x03\xA7{\xD4\"\x00\x00\x00\x02".b,
+    }.each do |int, payload|
+      expect(MessagePack::Bigint.to_msgpack_ext(int)).to be == payload
+      expect(MessagePack::Bigint.from_msgpack_ext(payload)).to be == int
+    end
+  end
+end

data/spec/factory_spec.rb

@@ -1,4 +1,3 @@
-# encoding: ascii-8bit
 require 'spec_helper'
 
 describe MessagePack::Factory do
@@ -42,6 +41,21 @@ describe MessagePack::Factory do
       unpacker.feed(MessagePack::ExtensionValue.new(1, 'a').to_msgpack)
       expect{ unpacker.read }.to raise_error(MessagePack::UnknownExtTypeError)
     end
+
+    it 'does not share the extension registry with unpackers' do
+      subject.register_type(0x00, Symbol)
+      expect do
+        unpacker = subject.unpacker
+        expect do
+          unpacker.register_type(0x01) {}
+        end.to change { unpacker.registered_types }
+
+        second_unpacker = subject.unpacker
+        expect do
+          second_unpacker.register_type(0x01) {}
+        end.to_not change { unpacker.registered_types }
+      end.to_not change { subject.registered_types }
+    end
   end
 
   describe '#dump and #load' do
@@ -255,29 +269,166 @@ describe MessagePack::Factory do
         subject { factory.packer.pack(value).to_s }
         before { stub_const('Value', Class.new{ include Mod }) }
         let(:value) { Value.new }
-        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
+        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked".force_encoding(Encoding::BINARY) }
       end
 
       describe "packing an object which has been extended by the module" do
         subject { factory.packer.pack(object).to_s }
         let(:object) { Object.new.extend Mod }
-        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
+        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked".force_encoding(Encoding::BINARY) }
       end
 
       describe "unpacking with the module" do
-        subject { factory.unpacker.feed("\xC7\x06\x01module").unpack }
+        subject { factory.unpacker.feed("\xC7\x06\x01module".force_encoding(Encoding::BINARY)).unpack }
         it { is_expected.to eq "unpacked module" }
       end
     end
+
+    describe "registering an ext type for Integer" do
+      let(:factory) { described_class.new }
+      let(:bigint) { 10**150 }
+
+      it 'does not work by default without passing `oversized_integer_extension: true`' do
+        factory.register_type(0x01, Integer, packer: :to_s, unpacker: method(:Integer))
+
+        expect do
+          factory.dump(bigint)
+        end.to raise_error RangeError
+      end
+
+      it 'raises ArgumentError if the type is not Integer' do
+        expect do
+          factory.register_type(0x01, MyType, packer: :to_s, unpacker: method(:Integer), oversized_integer_extension: true)
+        end.to raise_error(ArgumentError)
+      end
+
+      it 'invokes the packer if registered with `oversized_integer_extension: true`' do
+        factory.register_type(0x01, Integer, packer: :to_s, unpacker: method(:Integer), oversized_integer_extension: true)
+
+        expect(factory.load(factory.dump(bigint))).to be == bigint
+      end
+
+      it 'does not use the oversized_integer_extension packer for integers fitting in native types' do
+        factory.register_type(
+          0x01,
+          Integer,
+          packer: ->(int) { raise NotImplementedError },
+          unpacker: ->(payload) { raise NotImplementedError },
+          oversized_integer_extension: true
+        )
+
+        expect(factory.dump(42)).to eq(MessagePack.dump(42))
+      end
+    end
+
+    describe "registering ext type with recursive serialization" do
+      before do
+        stub_const("Point", Struct.new(:x, :y, :z))
+      end
+
+      it 'can receive the packer as argument (proc)' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_h)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump(point)
+        expect(factory.load(payload)).to be == point
+      end
+
+      it 'can receive the packer as argument (Method)' do
+        mod = Module.new
+        mod.define_singleton_method(:packer) do |point, packer|
+          packer.write(point.to_h)
+          nil
+        end
+
+        mod.define_singleton_method(:unpacker) do |unpacker|
+          attrs = unpacker.read
+          Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+        end
+
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: mod.method(:packer),
+          unpacker: mod.method(:unpacker),
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump(point)
+        expect(factory.load(payload)).to be == point
+      end
+
+      it 'respect message pack format' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_a)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(*attrs)
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        expect(factory.dump(point)).to be == "\xD6\x01".b + MessagePack.dump([1, 2, 3])
+      end
+
+      it 'sets the correct length' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_h)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump([1, point, 3])
+
+        obj = MessagePack::Factory.new.load(payload, allow_unknown_ext: true)
+        expect(obj).to be == [
+          1,
+          MessagePack::ExtensionValue.new(1, factory.dump(x: 1, y: 2, z: 3)),
+          3,
+        ]
+      end
+    end
   end
 
   describe 'the special treatment of symbols with ext type' do
-    let(:packer) { subject.packer }
-    let(:unpacker) { subject.unpacker }
-
-    def symbol_after_roundtrip
-      packed_symbol = packer.pack(:symbol).to_s
-      unpacker.feed(packed_symbol).unpack
+    def roundtrip(object, options = nil)
+      subject.load(subject.dump(object), options)
     end
 
     context 'using the optimized symbol unpacker' do
@@ -293,13 +444,25 @@ describe MessagePack::Factory do
       end
 
       it 'lets symbols survive a roundtrip' do
-        expect(symbol_after_roundtrip).to be :symbol
+        expect(roundtrip(:symbol)).to be :symbol
+      end
+
+      it 'preserves encoding for ASCII symbols' do
+        expect(:symbol.encoding).to be Encoding::US_ASCII
+        expect(roundtrip(:symbol)).to be :symbol
+        expect(roundtrip(:symbol).encoding).to be Encoding::US_ASCII
+      end
+
+      it 'preserves encoding for UTF-8 symbols' do
+        expect(:"fée".encoding).to be Encoding::UTF_8
+        expect(roundtrip(:"fée").encoding).to be Encoding::UTF_8
+        expect(roundtrip(:"fée")).to be :"fée"
       end
     end
 
     context 'if no ext type is registered for symbols' do
       it 'converts symbols to string' do
-        expect(symbol_after_roundtrip).to eq 'symbol'
+        expect(roundtrip(:symbol)).to eq 'symbol'
       end
     end
 
@@ -308,7 +471,41 @@ describe MessagePack::Factory do
         before { subject.register_type(0x00, ::Symbol) }
 
         it 'lets symbols survive a roundtrip' do
-          expect(symbol_after_roundtrip).to be :symbol
+          expect(roundtrip(:symbol)).to be :symbol
+        end
+
+        it 'works with hash keys' do
+          expect(roundtrip(symbol: 1)).to be == { symbol: 1 }
+        end
+
+        it 'works with frozen: true option' do
+          expect(roundtrip(:symbol, freeze: true)).to be :symbol
+        end
+
+        it 'preserves encoding for ASCII symbols' do
+          expect(:symbol.encoding).to be Encoding::US_ASCII
+          expect(roundtrip(:symbol)).to be :symbol
+          expect(roundtrip(:symbol).encoding).to be Encoding::US_ASCII
+        end
+
+        it 'preserves encoding for UTF-8 symbols' do
+          expect(:"fée".encoding).to be Encoding::UTF_8
+          expect(roundtrip(:"fée")).to be :"fée"
+          expect(roundtrip(:"fée").encoding).to be Encoding::UTF_8
+        end
+
+        it 'does not handle symbols in other encodings' do
+          symbol = "fàe".encode(Encoding::ISO_8859_1).to_sym
+          expect(symbol.encoding).to be Encoding::ISO_8859_1
+
+          if IS_JRUBY
+            # JRuby doesn't quite behave like MRI here.
+            # "fàe".force_encoding(Encoding::BINARY).to_sym is able to lookup the existing ISO-8859-1 symbol
+            # It likely is a JRuby bug.
+            expect(roundtrip(symbol).encoding).to be Encoding::ISO_8859_1
+          else
+            expect(roundtrip(symbol).encoding).to be Encoding::BINARY
+          end
         end
       end
 
@@ -332,7 +529,7 @@ describe MessagePack::Factory do
         before { subject.register_type(0x00, ::Symbol) }
 
         it 'lets symbols survive a roundtrip' do
-          expect(symbol_after_roundtrip).to be :symbol
+          expect(roundtrip(:symbol)).to be :symbol
         end
 
         after do
@@ -381,4 +578,56 @@ describe MessagePack::Factory do
       expect(MessagePack.unpack(MessagePack.pack(dm2))).to eq(dm2)
     end
   end
+
+  describe '#pool' do
+    let(:factory) { described_class.new }
+
+    it 'responds to serializers interface' do
+      pool = factory.pool(1)
+      expect(pool.load(pool.dump(42))).to be == 42
+    end
+
+    it 'types can be registered before the pool is created' do
+      factory.register_type(0x00, Symbol)
+      pool = factory.pool(1)
+      expect(pool.load(pool.dump(:foo))).to be == :foo
+    end
+
+    it 'types cannot be registered after the pool is created' do
+      pool = factory.pool(1)
+      factory.register_type(0x20, ::MyType)
+
+      expect do
+        pool.dump(MyType.new(1, 2))
+      end.to raise_error NoMethodError
+
+      payload = factory.dump(MyType.new(1, 2))
+      expect do
+        pool.load(payload)
+      end.to raise_error MessagePack::UnknownExtTypeError
+    end
+
+    it 'support symbolize_keys: true' do
+      pool = factory.pool(1, symbolize_keys: true)
+      expect(pool.load(pool.dump('foo' => 1))).to be == { foo: 1 }
+    end
+
+    it 'support freeze: true' do
+      pool = factory.pool(1, freeze: true)
+      expect(pool.load(pool.dump('foo'))).to be_frozen
+    end
+
+    it 'is thread safe' do
+      pool = factory.pool(1)
+
+      threads = 10.times.map do
+        Thread.new do
+          1_000.times do |i|
+            expect(pool.load(pool.dump(i))).to be == i
+          end
+        end
+      end
+      threads.each(&:join)
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -14,6 +14,7 @@ if ENV['GC_STRESS']
 end
 
 require 'msgpack'
+require "msgpack/bigint"
 
 if GC.respond_to?(:verify_compaction_references)
   # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
@@ -25,9 +26,7 @@ if GC.respond_to?(:auto_compact=)
   GC.auto_compact = true
 end
 
-def java?
-  /java/ =~ RUBY_PLATFORM
-end
+IS_JRUBY = RUBY_ENGINE == 'jruby'
 
 # checking if Hash#[]= (rb_hash_aset) dedupes string keys
 def automatic_string_keys_deduplication?
@@ -46,7 +45,7 @@ def string_deduplication?
   (-r1).equal?(-r2)
 end
 
-if java?
+if IS_JRUBY
   RSpec.configure do |c|
     c.treat_symbols_as_metadata_keys_with_true_values = true
     c.filter_run_excluding :encodings => !(defined? Encoding)