msgpack 1.2.10 → 1.5.1

data/lib/msgpack.rb

@@ -1,15 +1,10 @@
 require "msgpack/version"
 
 if defined?(RUBY_ENGINE) && RUBY_ENGINE == "jruby" # This is same with `/java/ =~ RUBY_VERSION`
-  require "java"
   require "msgpack/msgpack.jar"
-  org.msgpack.jruby.MessagePackLibrary.new.load(JRuby.runtime, false)
+  JRuby::Util.load_ext("org.msgpack.jruby.MessagePackLibrary")
 else
-  begin
-    require "msgpack/#{RUBY_VERSION[/\d+.\d+/]}/msgpack"
-  rescue LoadError
-    require "msgpack/msgpack"
-  end
+  require "msgpack/msgpack"
 end
 
 require "msgpack/packer"
@@ -17,19 +12,20 @@ require "msgpack/unpacker"
 require "msgpack/factory"
 require "msgpack/symbol"
 require "msgpack/core_ext"
+require "msgpack/timestamp"
+require "msgpack/time"
 
 module MessagePack
   DefaultFactory = MessagePack::Factory.new
-  DEFAULT_EMPTY_PARAMS = {}.freeze
 
   def load(src, param = nil)
     unpacker = nil
 
     if src.is_a? String
-      unpacker = DefaultFactory.unpacker param || DEFAULT_EMPTY_PARAMS
+      unpacker = DefaultFactory.unpacker param
       unpacker.feed_reference src
     else
-      unpacker = DefaultFactory.unpacker src, param || DEFAULT_EMPTY_PARAMS
+      unpacker = DefaultFactory.unpacker src, param
     end
 
     unpacker.full_unpack
@@ -39,8 +35,8 @@ module MessagePack
   module_function :load
   module_function :unpack
 
-  def pack(v, *rest)
-    packer = DefaultFactory.packer(*rest)
+  def pack(v, io = nil, options = nil)
+    packer = DefaultFactory.packer(io, options)
     packer.write v
     packer.full_pack
   end

data/msgpack.gemspec

@@ -10,7 +10,6 @@ Gem::Specification.new do |s|
   s.email = ["frsyuki@gmail.com", "theo@iconara.net", "tagomoris@gmail.com"]
   s.license = "Apache 2.0"
   s.homepage = "http://msgpack.org/"
-  s.rubyforge_project = "msgpack"
   s.require_paths = ["lib"]
   if /java/ =~ RUBY_PLATFORM
     s.files = Dir['lib/**/*.rb', 'lib/**/*.jar']
@@ -19,15 +18,12 @@ Gem::Specification.new do |s|
     s.files = `git ls-files`.split("\n")
     s.extensions = ["ext/msgpack/extconf.rb"]
   end
-  s.test_files = `git ls-files -- {test,spec}/*`.split("\n")
+
+  s.required_ruby_version = ">= 2.4"
 
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'rake'
-  s.add_development_dependency 'rake-compiler', ['~> 1.0']
-  if /java/ !~ RUBY_PLATFORM
-    # NOTE: rake-compiler-dock SHOULD be updated for new Ruby versions
-    s.add_development_dependency 'rake-compiler-dock', ['~> 0.7.0']
-  end
+  s.add_development_dependency 'rake-compiler', ['>= 1.1.9']
   s.add_development_dependency 'rspec', ['~> 3.3']
   s.add_development_dependency 'yard'
   s.add_development_dependency 'json'

data/spec/bigint_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe MessagePack::Bigint do
+  it 'serialize and deserialize arbitrary sized integer' do
+    [
+      1,
+      -1,
+      120938120391283122132313,
+      -21903120391203912391023920332103,
+      210290021321301203912933021323,
+    ].each do |int|
+      expect(MessagePack::Bigint.from_msgpack_ext(MessagePack::Bigint.to_msgpack_ext(int))).to be == int
+    end
+  end
+
+  it 'has a stable format' do
+    {
+      120938120391283122132313 => "\x00\x9F\xF4UY\x11\x92\x9A?\x00\x00\x19\x9C".b,
+      -21903120391203912391023920332103 => "\x01/\xB2\xBDG\xBD\xDE\xAA\xEBt\xCC\x8A\xC1\x00\x00\x01\x14".b,
+      210290021321301203912933021323 => "\x00\xC4\xD8\x96\x8Bm\xCB\xC7\x03\xA7{\xD4\"\x00\x00\x00\x02".b,
+    }.each do |int, payload|
+      expect(MessagePack::Bigint.to_msgpack_ext(int)).to be == payload
+      expect(MessagePack::Bigint.from_msgpack_ext(payload)).to be == int
+    end
+  end
+end

data/spec/factory_spec.rb

@@ -1,4 +1,3 @@
-# encoding: ascii-8bit
 require 'spec_helper'
 
 describe MessagePack::Factory do
@@ -42,6 +41,21 @@ describe MessagePack::Factory do
       unpacker.feed(MessagePack::ExtensionValue.new(1, 'a').to_msgpack)
       expect{ unpacker.read }.to raise_error(MessagePack::UnknownExtTypeError)
     end
+
+    it 'does not share the extension registry with unpackers' do
+      subject.register_type(0x00, Symbol)
+      expect do
+        unpacker = subject.unpacker
+        expect do
+          unpacker.register_type(0x01) {}
+        end.to change { unpacker.registered_types }
+
+        second_unpacker = subject.unpacker
+        expect do
+          second_unpacker.register_type(0x01) {}
+        end.to_not change { unpacker.registered_types }
+      end.to_not change { subject.registered_types }
+    end
   end
 
   describe '#dump and #load' do
@@ -255,34 +269,221 @@ describe MessagePack::Factory do
         subject { factory.packer.pack(value).to_s }
         before { stub_const('Value', Class.new{ include Mod }) }
         let(:value) { Value.new }
-        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
+        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked".force_encoding(Encoding::BINARY) }
       end
 
       describe "packing an object which has been extended by the module" do
         subject { factory.packer.pack(object).to_s }
         let(:object) { Object.new.extend Mod }
-        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
+        it { is_expected.to eq "\xC7\x0F\x01value_msgpacked".force_encoding(Encoding::BINARY) }
       end
 
       describe "unpacking with the module" do
-        subject { factory.unpacker.feed("\xC7\x06\x01module").unpack }
+        subject { factory.unpacker.feed("\xC7\x06\x01module".force_encoding(Encoding::BINARY)).unpack }
         it { is_expected.to eq "unpacked module" }
       end
     end
+
+    describe "registering an ext type for Integer" do
+      let(:factory) { described_class.new }
+      let(:bigint) { 10**150 }
+
+      it 'does not work by default without passing `oversized_integer_extension: true`' do
+        factory.register_type(0x01, Integer, packer: :to_s, unpacker: method(:Integer))
+
+        expect do
+          factory.dump(bigint)
+        end.to raise_error RangeError
+      end
+
+      it 'raises ArgumentError if the type is not Integer' do
+        expect do
+          factory.register_type(0x01, MyType, packer: :to_s, unpacker: method(:Integer), oversized_integer_extension: true)
+        end.to raise_error(ArgumentError)
+      end
+
+      it 'invokes the packer if registered with `oversized_integer_extension: true`' do
+        factory.register_type(0x01, Integer, packer: :to_s, unpacker: method(:Integer), oversized_integer_extension: true)
+
+        expect(factory.load(factory.dump(bigint))).to be == bigint
+      end
+
+      it 'does not use the oversized_integer_extension packer for integers fitting in native types' do
+        factory.register_type(
+          0x01,
+          Integer,
+          packer: ->(int) { raise NotImplementedError },
+          unpacker: ->(payload) { raise NotImplementedError },
+          oversized_integer_extension: true
+        )
+
+        expect(factory.dump(42)).to eq(MessagePack.dump(42))
+      end
+    end
+
+    describe "registering ext type with recursive serialization" do
+      before do
+        stub_const("Point", Struct.new(:x, :y, :z))
+      end
+
+      it 'can receive the packer as argument (proc)' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_h)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump(point)
+        expect(factory.load(payload)).to be == point
+      end
+
+      it 'can receive the packer as argument (Method)' do
+        mod = Module.new
+        mod.define_singleton_method(:packer) do |point, packer|
+          packer.write(point.to_h)
+          nil
+        end
+
+        mod.define_singleton_method(:unpacker) do |unpacker|
+          attrs = unpacker.read
+          Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+        end
+
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: mod.method(:packer),
+          unpacker: mod.method(:unpacker),
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump(point)
+        expect(factory.load(payload)).to be == point
+      end
+
+      it 'respect message pack format' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_a)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(*attrs)
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        expect(factory.dump(point)).to be == "\xD6\x01".b + MessagePack.dump([1, 2, 3])
+      end
+
+      it 'sets the correct length' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_h)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump([1, point, 3])
+
+        obj = MessagePack::Factory.new.load(payload, allow_unknown_ext: true)
+        expect(obj).to be == [
+          1,
+          MessagePack::ExtensionValue.new(1, factory.dump(x: 1, y: 2, z: 3)),
+          3,
+        ]
+      end
+
+      it 'can be nested' do
+        factory = MessagePack::Factory.new
+        factory.register_type(
+          0x02,
+          Set,
+          packer: ->(set, packer) do
+            packer.write(set.to_a)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            unpacker.read.to_set
+          end,
+          recursive: true,
+        )
+
+        expected = Set[1, Set[2, Set[3]]]
+        payload = factory.dump(expected)
+        expect(payload).to be == "\xC7\v\x02\x92\x01\xC7\x06\x02\x92\x02\xD5\x02\x91\x03".b
+        expect(factory.load(factory.dump(expected))).to be == expected
+      end
+    end
   end
 
   describe 'the special treatment of symbols with ext type' do
-    let(:packer) { subject.packer }
-    let(:unpacker) { subject.unpacker }
+    def roundtrip(object, options = nil)
+      subject.load(subject.dump(object), options)
+    end
 
-    def symbol_after_roundtrip
-      packed_symbol = packer.pack(:symbol).to_s
-      unpacker.feed(packed_symbol).unpack
+    context 'using the optimized symbol unpacker' do
+      before do
+        skip if IS_JRUBY # JRuby implementation doesn't support the optimized symbols unpacker for now
+        subject.register_type(
+          0x00,
+          ::Symbol,
+          packer: :to_msgpack_ext,
+          unpacker: :from_msgpack_ext,
+          optimized_symbols_parsing: true,
+        )
+      end
+
+      it 'lets symbols survive a roundtrip' do
+        expect(roundtrip(:symbol)).to be :symbol
+      end
+
+      it 'preserves encoding for ASCII symbols' do
+        expect(:symbol.encoding).to be Encoding::US_ASCII
+        expect(roundtrip(:symbol)).to be :symbol
+        expect(roundtrip(:symbol).encoding).to be Encoding::US_ASCII
+      end
+
+      it 'preserves encoding for UTF-8 symbols' do
+        expect(:"fée".encoding).to be Encoding::UTF_8
+        expect(roundtrip(:"fée").encoding).to be Encoding::UTF_8
+        expect(roundtrip(:"fée")).to be :"fée"
+      end
     end
 
     context 'if no ext type is registered for symbols' do
       it 'converts symbols to string' do
-        expect(symbol_after_roundtrip).to eq 'symbol'
+        expect(roundtrip(:symbol)).to eq 'symbol'
       end
     end
 
@@ -291,7 +492,41 @@ describe MessagePack::Factory do
         before { subject.register_type(0x00, ::Symbol) }
 
         it 'lets symbols survive a roundtrip' do
-          expect(symbol_after_roundtrip).to be :symbol
+          expect(roundtrip(:symbol)).to be :symbol
+        end
+
+        it 'works with hash keys' do
+          expect(roundtrip(symbol: 1)).to be == { symbol: 1 }
+        end
+
+        it 'works with frozen: true option' do
+          expect(roundtrip(:symbol, freeze: true)).to be :symbol
+        end
+
+        it 'preserves encoding for ASCII symbols' do
+          expect(:symbol.encoding).to be Encoding::US_ASCII
+          expect(roundtrip(:symbol)).to be :symbol
+          expect(roundtrip(:symbol).encoding).to be Encoding::US_ASCII
+        end
+
+        it 'preserves encoding for UTF-8 symbols' do
+          expect(:"fée".encoding).to be Encoding::UTF_8
+          expect(roundtrip(:"fée")).to be :"fée"
+          expect(roundtrip(:"fée").encoding).to be Encoding::UTF_8
+        end
+
+        it 'does not handle symbols in other encodings' do
+          symbol = "fàe".encode(Encoding::ISO_8859_1).to_sym
+          expect(symbol.encoding).to be Encoding::ISO_8859_1
+
+          if IS_JRUBY
+            # JRuby doesn't quite behave like MRI here.
+            # "fàe".force_encoding(Encoding::BINARY).to_sym is able to lookup the existing ISO-8859-1 symbol
+            # It likely is a JRuby bug.
+            expect(roundtrip(symbol).encoding).to be Encoding::ISO_8859_1
+          else
+            expect(roundtrip(symbol).encoding).to be Encoding::BINARY
+          end
         end
       end
 
@@ -315,7 +550,7 @@ describe MessagePack::Factory do
         before { subject.register_type(0x00, ::Symbol) }
 
         it 'lets symbols survive a roundtrip' do
-          expect(symbol_after_roundtrip).to be :symbol
+          expect(roundtrip(:symbol)).to be :symbol
         end
 
         after do
@@ -364,4 +599,56 @@ describe MessagePack::Factory do
       expect(MessagePack.unpack(MessagePack.pack(dm2))).to eq(dm2)
     end
   end
+
+  describe '#pool' do
+    let(:factory) { described_class.new }
+
+    it 'responds to serializers interface' do
+      pool = factory.pool(1)
+      expect(pool.load(pool.dump(42))).to be == 42
+    end
+
+    it 'types can be registered before the pool is created' do
+      factory.register_type(0x00, Symbol)
+      pool = factory.pool(1)
+      expect(pool.load(pool.dump(:foo))).to be == :foo
+    end
+
+    it 'types cannot be registered after the pool is created' do
+      pool = factory.pool(1)
+      factory.register_type(0x20, ::MyType)
+
+      expect do
+        pool.dump(MyType.new(1, 2))
+      end.to raise_error NoMethodError
+
+      payload = factory.dump(MyType.new(1, 2))
+      expect do
+        pool.load(payload)
+      end.to raise_error MessagePack::UnknownExtTypeError
+    end
+
+    it 'support symbolize_keys: true' do
+      pool = factory.pool(1, symbolize_keys: true)
+      expect(pool.load(pool.dump('foo' => 1))).to be == { foo: 1 }
+    end
+
+    it 'support freeze: true' do
+      pool = factory.pool(1, freeze: true)
+      expect(pool.load(pool.dump('foo'))).to be_frozen
+    end
+
+    it 'is thread safe' do
+      pool = factory.pool(1)
+
+      threads = 10.times.map do
+        Thread.new do
+          1_000.times do |i|
+            expect(pool.load(pool.dump(i))).to be == i
+          end
+        end
+      end
+      threads.each(&:join)
+    end
+  end
 end

data/spec/msgpack_spec.rb

@@ -115,7 +115,7 @@ describe MessagePack do
       expect { MessagePack.pack(self) }.to raise_error(NoMethodError, /^undefined method `to_msgpack'/)
     end
 
-    it 'rasies an error on #unpack with garbage' do
+    it 'raises an error on #unpack with garbage' do
       skip "but nothing was raised. why?"
       expect { MessagePack.unpack('asdka;sd') }.to raise_error(MessagePack::UnpackError)
     end

data/spec/packer_spec.rb

@@ -488,6 +488,24 @@ describe MessagePack::Packer do
       it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
     end
 
+    shared_examples_for 'extension subclasses core type' do |klass|
+      before { stub_const('Value', Class.new(klass)) }
+      let(:object) { Value.new }
+      subject { packer.pack(object).to_s }
+
+      it "defaults to #{klass.name} packer if no extension is present" do
+        expect(subject).to eq(MessagePack.dump(klass.new))
+      end
+
+      it "uses core type extension for #{klass.name}" do
+        packer.register_type(0x01, Value, ->(_) { 'value_msgpacked' })
+        expect(subject).to eq("\xC7\x0F\x01value_msgpacked")
+      end
+    end
+    it_behaves_like 'extension subclasses core type', Hash
+    it_behaves_like 'extension subclasses core type', Array
+    it_behaves_like 'extension subclasses core type', String
+
     context 'when registering a type for symbols' do
       before { packer.register_type(0x00, ::Symbol, :to_msgpack_ext) }
 

data/spec/spec_helper.rb

@@ -1,3 +1,4 @@
+require "set"
 
 if ENV['SIMPLE_COV']
   require 'simplecov'
@@ -14,12 +15,38 @@ if ENV['GC_STRESS']
 end
 
 require 'msgpack'
+require "msgpack/bigint"
 
-def java?
-  /java/ =~ RUBY_PLATFORM
+if GC.respond_to?(:verify_compaction_references)
+  # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
+  # move objects around, helping to find object movement bugs.
+  GC.verify_compaction_references(double_heap: true, toward: :empty)
 end
 
-if java?
+if GC.respond_to?(:auto_compact=)
+  GC.auto_compact = true
+end
+
+IS_JRUBY = RUBY_ENGINE == 'jruby'
+
+# checking if Hash#[]= (rb_hash_aset) dedupes string keys
+def automatic_string_keys_deduplication?
+  h = {}
+  x = {}
+  r = rand.to_s
+  h[%W(#{r}).join('')] = :foo
+  x[%W(#{r}).join('')] = :foo
+
+  x.keys[0].equal?(h.keys[0])
+end
+
+def string_deduplication?
+  r1 = rand.to_s
+  r2 = r1.dup
+  (-r1).equal?(-r2)
+end
+
+if IS_JRUBY
   RSpec.configure do |c|
     c.treat_symbols_as_metadata_keys_with_true_values = true
     c.filter_run_excluding :encodings => !(defined? Encoding)