msgpack 1.3.0 → 1.6.0

data/spec/factory_spec.rb

@@ -1,4 +1,3 @@
-# encoding: ascii-8bit
 require 'spec_helper'
 
 describe MessagePack::Factory do
@@ -42,6 +41,21 @@ describe MessagePack::Factory do
       unpacker.feed(MessagePack::ExtensionValue.new(1, 'a').to_msgpack)
       expect{ unpacker.read }.to raise_error(MessagePack::UnknownExtTypeError)
     end
+
+    it 'does not share the extension registry with unpackers' do
+      subject.register_type(0x00, Symbol)
+      expect do
+        unpacker = subject.unpacker
+        expect do
+          unpacker.register_type(0x01) {}
+        end.to change { unpacker.registered_types }
+
+        second_unpacker = subject.unpacker
+        expect do
+          second_unpacker.register_type(0x01) {}
+        end.to_not change { unpacker.registered_types }
+      end.to_not change { subject.registered_types }
+    end
   end
 
   describe '#dump and #load' do
@@ -255,34 +269,231 @@ describe MessagePack::Factory do
         subject { factory.packer.pack(value).to_s }
         before { stub_const('Value', Class.new{ include Mod }) }
         let(:value) { Value.new }
-        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
+        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked".force_encoding(Encoding::BINARY) }
       end
 
       describe "packing an object which has been extended by the module" do
         subject { factory.packer.pack(object).to_s }
         let(:object) { Object.new.extend Mod }
-        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
+        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked".force_encoding(Encoding::BINARY) }
       end
 
       describe "unpacking with the module" do
-        subject { factory.unpacker.feed("\xC7\x06\x01module").unpack }
+        subject { factory.unpacker.feed("\xC7\x06\x01module".force_encoding(Encoding::BINARY)).unpack }
         it { is_expected.to eq "unpacked module" }
       end
     end
+
+    describe "registering an ext type for Integer" do
+      let(:factory) { described_class.new }
+      let(:bigint) { 10**150 }
+
+      it 'does not work by default without passing `oversized_integer_extension: true`' do
+        factory.register_type(0x01, Integer, packer: :to_s, unpacker: method(:Integer))
+
+        expect do
+          factory.dump(bigint)
+        end.to raise_error RangeError
+      end
+
+      it 'raises ArgumentError if the type is not Integer' do
+        expect do
+          factory.register_type(0x01, MyType, packer: :to_s, unpacker: method(:Integer), oversized_integer_extension: true)
+        end.to raise_error(ArgumentError)
+      end
+
+      it 'invokes the packer if registered with `oversized_integer_extension: true`' do
+        factory.register_type(0x01, Integer, packer: :to_s, unpacker: method(:Integer), oversized_integer_extension: true)
+
+        expect(factory.load(factory.dump(bigint))).to be == bigint
+      end
+
+      it 'does not use the oversized_integer_extension packer for integers fitting in native types' do
+        factory.register_type(
+          0x01,
+          Integer,
+          packer: ->(int) { raise NotImplementedError },
+          unpacker: ->(payload) { raise NotImplementedError },
+          oversized_integer_extension: true
+        )
+
+        expect(factory.dump(42)).to eq(MessagePack.dump(42))
+      end
+    end
+
+    describe "registering ext type with recursive serialization" do
+      before do
+        stub_const("Point", Struct.new(:x, :y, :z))
+      end
+
+      it 'can receive the packer as argument (proc)' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_h)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump(point)
+        expect(factory.load(payload)).to be == point
+      end
+
+      it 'can receive the packer as argument (Method)' do
+        mod = Module.new
+        mod.define_singleton_method(:packer) do |point, packer|
+          packer.write(point.to_h)
+          nil
+        end
+
+        mod.define_singleton_method(:unpacker) do |unpacker|
+          attrs = unpacker.read
+          Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+        end
+
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: mod.method(:packer),
+          unpacker: mod.method(:unpacker),
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump(point)
+        expect(factory.load(payload)).to be == point
+      end
+
+      it 'respect message pack format' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_a)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(*attrs)
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        expect(factory.dump(point)).to be == "\xD6\x01".b + MessagePack.dump([1, 2, 3])
+      end
+
+      it 'sets the correct length' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_h)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump([1, point, 3])
+
+        obj = MessagePack::Factory.new.load(payload, allow_unknown_ext: true)
+        expect(obj).to be == [
+          1,
+          MessagePack::ExtensionValue.new(1, factory.dump(x: 1, y: 2, z: 3)),
+          3,
+        ]
+
+        expect(factory.load(payload)).to be == [
+          1,
+          Point.new(1, 2, 3),
+          3,
+        ]
+      end
+
+      it 'can be nested' do
+        factory = MessagePack::Factory.new
+        factory.register_type(
+          0x02,
+          Set,
+          packer: ->(set, packer) do
+            packer.write(set.to_a)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            unpacker.read.to_set
+          end,
+          recursive: true,
+        )
+
+        expected = Set[1, Set[2, Set[3]]]
+        payload = factory.dump(expected)
+        expect(payload).to be == "\xC7\v\x02\x92\x01\xC7\x06\x02\x92\x02\xD5\x02\x91\x03".b
+        expect(factory.load(factory.dump(expected))).to be == expected
+      end
+    end
   end
 
   describe 'the special treatment of symbols with ext type' do
-    let(:packer) { subject.packer }
-    let(:unpacker) { subject.unpacker }
+    def roundtrip(object, options = nil)
+      subject.load(subject.dump(object), options)
+    end
 
-    def symbol_after_roundtrip
-      packed_symbol = packer.pack(:symbol).to_s
-      unpacker.feed(packed_symbol).unpack
+    context 'using the optimized symbol unpacker' do
+      before do
+        skip if IS_JRUBY # JRuby implementation doesn't support the optimized symbols unpacker for now
+        subject.register_type(
+          0x00,
+          ::Symbol,
+          packer: :to_msgpack_ext,
+          unpacker: :from_msgpack_ext,
+          optimized_symbols_parsing: true,
+        )
+      end
+
+      it 'lets symbols survive a roundtrip' do
+        expect(roundtrip(:symbol)).to be :symbol
+      end
+
+      it 'works with empty symbol' do
+        expect(roundtrip(:"")).to be :""
+      end
+
+      it 'preserves encoding for ASCII symbols' do
+        expect(:symbol.encoding).to be Encoding::US_ASCII
+        expect(roundtrip(:symbol)).to be :symbol
+        expect(roundtrip(:symbol).encoding).to be Encoding::US_ASCII
+      end
+
+      it 'preserves encoding for UTF-8 symbols' do
+        expect(:"fée".encoding).to be Encoding::UTF_8
+        expect(roundtrip(:"fée").encoding).to be Encoding::UTF_8
+        expect(roundtrip(:"fée")).to be :"fée"
+      end
     end
 
     context 'if no ext type is registered for symbols' do
       it 'converts symbols to string' do
-        expect(symbol_after_roundtrip).to eq 'symbol'
+        expect(roundtrip(:symbol)).to eq 'symbol'
       end
     end
 
@@ -291,7 +502,41 @@ describe MessagePack::Factory do
         before { subject.register_type(0x00, ::Symbol) }
 
         it 'lets symbols survive a roundtrip' do
-          expect(symbol_after_roundtrip).to be :symbol
+          expect(roundtrip(:symbol)).to be :symbol
+        end
+
+        it 'works with hash keys' do
+          expect(roundtrip(symbol: 1)).to be == { symbol: 1 }
+        end
+
+        it 'works with frozen: true option' do
+          expect(roundtrip(:symbol, freeze: true)).to be :symbol
+        end
+
+        it 'preserves encoding for ASCII symbols' do
+          expect(:symbol.encoding).to be Encoding::US_ASCII
+          expect(roundtrip(:symbol)).to be :symbol
+          expect(roundtrip(:symbol).encoding).to be Encoding::US_ASCII
+        end
+
+        it 'preserves encoding for UTF-8 symbols' do
+          expect(:"fée".encoding).to be Encoding::UTF_8
+          expect(roundtrip(:"fée")).to be :"fée"
+          expect(roundtrip(:"fée").encoding).to be Encoding::UTF_8
+        end
+
+        it 'does not handle symbols in other encodings' do
+          symbol = "fàe".encode(Encoding::ISO_8859_1).to_sym
+          expect(symbol.encoding).to be Encoding::ISO_8859_1
+
+          if IS_JRUBY
+            # JRuby doesn't quite behave like MRI here.
+            # "fàe".force_encoding(Encoding::BINARY).to_sym is able to lookup the existing ISO-8859-1 symbol
+            # It likely is a JRuby bug.
+            expect(roundtrip(symbol).encoding).to be Encoding::ISO_8859_1
+          else
+            expect(roundtrip(symbol).encoding).to be Encoding::BINARY
+          end
         end
       end
 
@@ -315,7 +560,7 @@ describe MessagePack::Factory do
         before { subject.register_type(0x00, ::Symbol) }
 
         it 'lets symbols survive a roundtrip' do
-          expect(symbol_after_roundtrip).to be :symbol
+          expect(roundtrip(:symbol)).to be :symbol
         end
 
         after do
@@ -342,6 +587,48 @@ describe MessagePack::Factory do
         GC.stress = false
       end
     end
+
+    it 'does not crash in recursive extensions' do
+      my_hash_type = Class.new(Hash)
+      factory = MessagePack::Factory.new
+      factory.register_type(7,
+        my_hash_type,
+        packer: ->(value, packer) do
+          packer.write(value.to_h)
+        end,
+        unpacker: ->(unpacker) { my_hash_type.new(unpacker.read) },
+        recursive: true,
+      )
+
+      payload = factory.dump(
+        [my_hash_type.new]
+      )
+
+      begin
+        GC.stress = true
+        factory.load(payload)
+      ensure
+        GC.stress = false
+      end
+    end
+  end
+
+  describe 'memsize' do
+    it 'works on a fresh factory' do
+      skip "JRuby doesn't support ObjectSpace.memsize_of" if IS_JRUBY
+
+      f = MessagePack::Factory.new
+      expect(ObjectSpace.memsize_of(f)).to be_an(Integer)
+    end
+
+    it 'works on a factory with registered types' do
+      skip "JRuby doesn't support ObjectSpace.memsize_of" if IS_JRUBY
+
+      f = MessagePack::Factory.new
+      base_size = ObjectSpace.memsize_of(f)
+      f.register_type(0x0a, Symbol)
+      expect(ObjectSpace.memsize_of(f)).to be > base_size
+    end
   end
 
   describe 'DefaultFactory' do
@@ -364,4 +651,56 @@ describe MessagePack::Factory do
       expect(MessagePack.unpack(MessagePack.pack(dm2))).to eq(dm2)
     end
   end
+
+  describe '#pool' do
+    let(:factory) { described_class.new }
+
+    it 'responds to serializers interface' do
+      pool = factory.pool(1)
+      expect(pool.load(pool.dump(42))).to be == 42
+    end
+
+    it 'types can be registered before the pool is created' do
+      factory.register_type(0x00, Symbol)
+      pool = factory.pool(1)
+      expect(pool.load(pool.dump(:foo))).to be == :foo
+    end
+
+    it 'types cannot be registered after the pool is created' do
+      pool = factory.pool(1)
+      factory.register_type(0x20, ::MyType)
+
+      expect do
+        pool.dump(MyType.new(1, 2))
+      end.to raise_error NoMethodError
+
+      payload = factory.dump(MyType.new(1, 2))
+      expect do
+        pool.load(payload)
+      end.to raise_error MessagePack::UnknownExtTypeError
+    end
+
+    it 'support symbolize_keys: true' do
+      pool = factory.pool(1, symbolize_keys: true)
+      expect(pool.load(pool.dump('foo' => 1))).to be == { foo: 1 }
+    end
+
+    it 'support freeze: true' do
+      pool = factory.pool(1, freeze: true)
+      expect(pool.load(pool.dump('foo'))).to be_frozen
+    end
+
+    it 'is thread safe' do
+      pool = factory.pool(1)
+
+      threads = 10.times.map do
+        Thread.new do
+          1_000.times do |i|
+            expect(pool.load(pool.dump(i))).to be == i
+          end
+        end
+      end
+      threads.each(&:join)
+    end
+  end
 end

data/spec/msgpack_spec.rb

@@ -115,7 +115,7 @@ describe MessagePack do
       expect { MessagePack.pack(self) }.to raise_error(NoMethodError, /^undefined method `to_msgpack'/)
     end
 
-    it 'rasies an error on #unpack with garbage' do
+    it 'raises an error on #unpack with garbage' do
       skip "but nothing was raised. why?"
       expect { MessagePack.unpack('asdka;sd') }.to raise_error(MessagePack::UnpackError)
     end

data/spec/packer_spec.rb

@@ -488,6 +488,24 @@ describe MessagePack::Packer do
       it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
     end
 
+    shared_examples_for 'extension subclasses core type' do |klass|
+      before { stub_const('Value', Class.new(klass)) }
+      let(:object) { Value.new }
+      subject { packer.pack(object).to_s }
+
+      it "defaults to #{klass.name} packer if no extension is present" do
+        expect(subject).to eq(MessagePack.dump(klass.new))
+      end
+
+      it "uses core type extension for #{klass.name}" do
+        packer.register_type(0x01, Value, ->(_) { 'value_msgpacked' })
+        expect(subject).to eq("\xC7\x0F\x01value_msgpacked")
+      end
+    end
+    it_behaves_like 'extension subclasses core type', Hash
+    it_behaves_like 'extension subclasses core type', Array
+    it_behaves_like 'extension subclasses core type', String
+
     context 'when registering a type for symbols' do
       before { packer.register_type(0x00, ::Symbol, :to_msgpack_ext) }
 

data/spec/spec_helper.rb

@@ -1,3 +1,5 @@
+require "set"
+require "objspace"
 
 if ENV['SIMPLE_COV']
   require 'simplecov'
@@ -14,12 +16,44 @@ if ENV['GC_STRESS']
 end
 
 require 'msgpack'
+require "msgpack/bigint"
 
-def java?
-  /java/ =~ RUBY_PLATFORM
+if GC.respond_to?(:verify_compaction_references)
+  # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
+  # move objects around, helping to find object movement bugs.
+  begin
+    GC.verify_compaction_references(double_heap: true, toward: :empty)
+  rescue NotImplementedError
+    # Some platforms don't support compaction
+  end
+end
+
+if GC.respond_to?(:auto_compact=)
+  GC.auto_compact = true
+end
+
+IS_JRUBY = RUBY_ENGINE == 'jruby'
+
+IS_TRUFFLERUBY = RUBY_ENGINE == 'truffleruby'
+
+# checking if Hash#[]= (rb_hash_aset) dedupes string keys
+def automatic_string_keys_deduplication?
+  h = {}
+  x = {}
+  r = rand.to_s
+  h[%W(#{r}).join('')] = :foo
+  x[%W(#{r}).join('')] = :foo
+
+  x.keys[0].equal?(h.keys[0])
+end
+
+def string_deduplication?
+  r1 = rand.to_s
+  r2 = r1.dup
+  (-r1).equal?(-r2)
 end
 
-if java?
+if IS_JRUBY
   RSpec.configure do |c|
     c.treat_symbols_as_metadata_keys_with_true_values = true
     c.filter_run_excluding :encodings => !(defined? Encoding)

data/spec/timestamp_spec.rb

@@ -56,6 +56,48 @@ describe MessagePack::Timestamp do
       expect(packed).to start_with(prefix_ext8_with_12bytes_payload_and_type_id)
       expect(packed.size).to eq(15)
     end
+
+    it 'runs correctly (regression)' do
+      expect(factory.unpack(factory.pack(Time.utc(2200)))).to eq(Time.utc(2200))
+    end
+
+    let(:time32_max) { Time.new(2106, 2, 7, 6, 28, 15, "+00:00") }
+    it 'is serialized into timestamp32' do
+      expect(factory.pack(time32_max).size).to be 6
+      expect(factory.unpack(factory.pack(time32_max)).utc).to eq(time32_max)
+    end
+
+    let(:time64_min) { Time.new(2106, 2, 7, 6, 28, 16, "+00:00") }
+    it 'is serialized into timestamp64' do
+      expect(factory.pack(time64_min).size).to be 10
+      expect(factory.unpack(factory.pack(time64_min)).utc).to eq(time64_min)
+    end
+
+    let(:time64_max) { Time.at(Time.new(2514, 5, 30, 1, 53, 3, "+00:00").to_i, 999999999 / 1000.0r).utc } # TODO: use Time.at(sec, nsec, :nsec) when removing Ruby 2.4 from the list
+    it 'is serialized into timestamp64' do
+      expect(factory.pack(time64_max).size).to be 10
+      expect(factory.unpack(factory.pack(time64_max)).utc).to eq(time64_max)
+    end
+
+    let(:time96_positive_min) { Time.new(2514, 5, 30, 1, 53, 4, "+00:00") }
+    it 'is serialized into timestamp96' do
+      expect(factory.pack(time96_positive_min).size).to be 15
+      expect(factory.unpack(factory.pack(time96_positive_min)).utc).to eq(time96_positive_min)
+    end
+
+    let(:time96_min) { Time.at(-2**63).utc }
+    it 'is serialized into timestamp96' do
+      skip if IS_JRUBY || IS_TRUFFLERUBY # JRuby and TruffleRuby both use underlying Java time classes that do not support |year| >= 1 billion
+      expect(factory.pack(time96_min).size).to be 15
+      expect(factory.unpack(factory.pack(time96_min)).utc).to eq(time96_min)
+    end
+
+    let(:time96_max) { Time.at(2**63 - 1).utc }
+    it 'is serialized into timestamp96' do
+      skip if IS_JRUBY || IS_TRUFFLERUBY # JRuby and TruffleRuby both use underlying Java time classes that do not support |year| >= 1 billion
+      expect(factory.pack(time96_max).size).to be 15
+      expect(factory.unpack(factory.pack(time96_max)).utc).to eq(time96_max)
+    end
   end
 
   describe 'register_type with MessagePack::Timestamp' do