msgpack 1.4.5 → 1.5.6

data/lib/msgpack/factory.rb

@@ -77,5 +77,108 @@ module MessagePack
       packer.full_pack
     end
     alias :pack :dump
+
+    def pool(size = 1, **options)
+      Pool.new(
+        frozen? ? self : dup.freeze,
+        size,
+        options.empty? ? nil : options,
+      )
+    end
+
+    class Pool
+      if RUBY_ENGINE == "ruby"
+        class AbstractPool
+          def initialize(size, &block)
+            @size = size
+            @new_member = block
+            @members = []
+          end
+
+          def checkout
+            @members.pop || @new_member.call
+          end
+
+          def checkin(member)
+            # If the pool is already full, we simply drop the extra member.
+            # This is because contrary to a connection pool, creating an extra instance
+            # is extremely unlikely to cause some kind of resource exhaustion.
+            #
+            # We could cycle the members (keep the newer one) but first It's more work and second
+            # the older member might have been created pre-fork, so it might be at least partially
+            # in shared memory.
+            if member && @members.size < @size
+              member.reset
+              @members << member
+            end
+          end
+        end
+      else
+        class AbstractPool
+          def initialize(size, &block)
+            @size = size
+            @new_member = block
+            @members = []
+            @mutex = Mutex.new
+          end
+
+          def checkout
+            @mutex.synchronize { @members.pop } || @new_member.call
+          end
+
+          def checkin(member)
+            @mutex.synchronize do
+              if member && @members.size < @size
+                member.reset
+                @members << member
+              end
+            end
+          end
+        end
+      end
+
+      class PackerPool < AbstractPool
+        private
+
+        def reset(packer)
+          packer.clear
+        end
+      end
+
+      class UnpackerPool < AbstractPool
+        private
+
+        def reset(unpacker)
+          unpacker.reset
+        end
+      end
+
+      def initialize(factory, size, options = nil)
+        options = nil if !options || options.empty?
+        @factory = factory
+        @packers = PackerPool.new(size) { factory.packer(options) }
+        @unpackers = UnpackerPool.new(size) { factory.unpacker(options) }
+      end
+
+      def load(data)
+        unpacker = @unpackers.checkout
+        begin
+          unpacker.feed_reference(data)
+          unpacker.full_unpack
+        ensure
+          @unpackers.checkin(unpacker)
+        end
+      end
+
+      def dump(object)
+        packer = @packers.checkout
+        begin
+          packer.write(object)
+          packer.full_pack
+        ensure
+          @packers.checkin(packer)
+        end
+      end
+    end
   end
 end

data/lib/msgpack/version.rb

@@ -1,5 +1,5 @@
 module MessagePack
-  VERSION = "1.4.5"
+  VERSION = "1.5.6"
   # Note for maintainers:
   #  Don't miss building/releasing the JRuby version (rake buld:java)
   #  See "How to build -java rubygems" in README for more details.

data/lib/msgpack.rb

@@ -17,16 +17,15 @@ require "msgpack/time"
 
 module MessagePack
   DefaultFactory = MessagePack::Factory.new
-  DEFAULT_EMPTY_PARAMS = {}.freeze
 
   def load(src, param = nil)
     unpacker = nil
 
     if src.is_a? String
-      unpacker = DefaultFactory.unpacker param || DEFAULT_EMPTY_PARAMS
+      unpacker = DefaultFactory.unpacker param
       unpacker.feed_reference src
     else
-      unpacker = DefaultFactory.unpacker src, param || DEFAULT_EMPTY_PARAMS
+      unpacker = DefaultFactory.unpacker src, param
     end
 
     unpacker.full_unpack
@@ -36,8 +35,8 @@ module MessagePack
   module_function :load
   module_function :unpack
 
-  def pack(v, *rest)
-    packer = DefaultFactory.packer(*rest)
+  def pack(v, io = nil, options = nil)
+    packer = DefaultFactory.packer(io, options)
     packer.write v
     packer.full_pack
   end

data/msgpack.gemspec

@@ -27,4 +27,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rspec', ['~> 3.3']
   s.add_development_dependency 'yard'
   s.add_development_dependency 'json'
+  s.add_development_dependency 'benchmark-ips', ['~> 2.10.0']
 end

data/spec/bigint_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe MessagePack::Bigint do
+  it 'serialize and deserialize arbitrary sized integer' do
+    [
+      1,
+      -1,
+      120938120391283122132313,
+      -21903120391203912391023920332103,
+      210290021321301203912933021323,
+    ].each do |int|
+      expect(MessagePack::Bigint.from_msgpack_ext(MessagePack::Bigint.to_msgpack_ext(int))).to be == int
+    end
+  end
+
+  it 'has a stable format' do
+    {
+      120938120391283122132313 => "\x00\x9F\xF4UY\x11\x92\x9A?\x00\x00\x19\x9C".b,
+      -21903120391203912391023920332103 => "\x01/\xB2\xBDG\xBD\xDE\xAA\xEBt\xCC\x8A\xC1\x00\x00\x01\x14".b,
+      210290021321301203912933021323 => "\x00\xC4\xD8\x96\x8Bm\xCB\xC7\x03\xA7{\xD4\"\x00\x00\x00\x02".b,
+    }.each do |int, payload|
+      expect(MessagePack::Bigint.to_msgpack_ext(int)).to be == payload
+      expect(MessagePack::Bigint.from_msgpack_ext(payload)).to be == int
+    end
+  end
+end

data/spec/factory_spec.rb

@@ -283,6 +283,174 @@ describe MessagePack::Factory do
         it { is_expected.to eq "unpacked module" }
       end
     end
+
+    describe "registering an ext type for Integer" do
+      let(:factory) { described_class.new }
+      let(:bigint) { 10**150 }
+
+      it 'does not work by default without passing `oversized_integer_extension: true`' do
+        factory.register_type(0x01, Integer, packer: :to_s, unpacker: method(:Integer))
+
+        expect do
+          factory.dump(bigint)
+        end.to raise_error RangeError
+      end
+
+      it 'raises ArgumentError if the type is not Integer' do
+        expect do
+          factory.register_type(0x01, MyType, packer: :to_s, unpacker: method(:Integer), oversized_integer_extension: true)
+        end.to raise_error(ArgumentError)
+      end
+
+      it 'invokes the packer if registered with `oversized_integer_extension: true`' do
+        factory.register_type(0x01, Integer, packer: :to_s, unpacker: method(:Integer), oversized_integer_extension: true)
+
+        expect(factory.load(factory.dump(bigint))).to be == bigint
+      end
+
+      it 'does not use the oversized_integer_extension packer for integers fitting in native types' do
+        factory.register_type(
+          0x01,
+          Integer,
+          packer: ->(int) { raise NotImplementedError },
+          unpacker: ->(payload) { raise NotImplementedError },
+          oversized_integer_extension: true
+        )
+
+        expect(factory.dump(42)).to eq(MessagePack.dump(42))
+      end
+    end
+
+    describe "registering ext type with recursive serialization" do
+      before do
+        stub_const("Point", Struct.new(:x, :y, :z))
+      end
+
+      it 'can receive the packer as argument (proc)' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_h)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump(point)
+        expect(factory.load(payload)).to be == point
+      end
+
+      it 'can receive the packer as argument (Method)' do
+        mod = Module.new
+        mod.define_singleton_method(:packer) do |point, packer|
+          packer.write(point.to_h)
+          nil
+        end
+
+        mod.define_singleton_method(:unpacker) do |unpacker|
+          attrs = unpacker.read
+          Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+        end
+
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: mod.method(:packer),
+          unpacker: mod.method(:unpacker),
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump(point)
+        expect(factory.load(payload)).to be == point
+      end
+
+      it 'respect message pack format' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_a)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(*attrs)
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        expect(factory.dump(point)).to be == "\xD6\x01".b + MessagePack.dump([1, 2, 3])
+      end
+
+      it 'sets the correct length' do
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        factory.register_type(
+          0x01,
+          Point,
+          packer: ->(point, packer) do
+            packer.write(point.to_h)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            attrs = unpacker.read
+            Point.new(attrs.fetch(:x), attrs.fetch(:y), attrs.fetch(:z))
+          end,
+          recursive: true,
+        )
+
+        point = Point.new(1, 2, 3)
+        payload = factory.dump([1, point, 3])
+
+        obj = MessagePack::Factory.new.load(payload, allow_unknown_ext: true)
+        expect(obj).to be == [
+          1,
+          MessagePack::ExtensionValue.new(1, factory.dump(x: 1, y: 2, z: 3)),
+          3,
+        ]
+
+        expect(factory.load(payload)).to be == [
+          1,
+          Point.new(1, 2, 3),
+          3,
+        ]
+      end
+
+      it 'can be nested' do
+        factory = MessagePack::Factory.new
+        factory.register_type(
+          0x02,
+          Set,
+          packer: ->(set, packer) do
+            packer.write(set.to_a)
+            nil
+          end,
+          unpacker: ->(unpacker) do
+            unpacker.read.to_set
+          end,
+          recursive: true,
+        )
+
+        expected = Set[1, Set[2, Set[3]]]
+        payload = factory.dump(expected)
+        expect(payload).to be == "\xC7\v\x02\x92\x01\xC7\x06\x02\x92\x02\xD5\x02\x91\x03".b
+        expect(factory.load(factory.dump(expected))).to be == expected
+      end
+    end
   end
 
   describe 'the special treatment of symbols with ext type' do
@@ -306,6 +474,10 @@ describe MessagePack::Factory do
         expect(roundtrip(:symbol)).to be :symbol
       end
 
+      it 'works with empty symbol' do
+        expect(roundtrip(:"")).to be :""
+      end
+
       it 'preserves encoding for ASCII symbols' do
         expect(:symbol.encoding).to be Encoding::US_ASCII
         expect(roundtrip(:symbol)).to be :symbol
@@ -415,6 +587,30 @@ describe MessagePack::Factory do
         GC.stress = false
       end
     end
+
+    it 'does not crash in recursive extensions' do
+      my_hash_type = Class.new(Hash)
+      factory = MessagePack::Factory.new
+      factory.register_type(7,
+        my_hash_type,
+        packer: ->(value, packer) do
+          packer.write(value.to_h)
+        end,
+        unpacker: ->(unpacker) { my_hash_type.new(unpacker.read) },
+        recursive: true,
+      )
+
+      payload = factory.dump(
+        [my_hash_type.new]
+      )
+
+      begin
+        GC.stress = true
+        factory.load(payload)
+      ensure
+        GC.stress = false
+      end
+    end
   end
 
   describe 'DefaultFactory' do
@@ -437,4 +633,56 @@ describe MessagePack::Factory do
       expect(MessagePack.unpack(MessagePack.pack(dm2))).to eq(dm2)
     end
   end
+
+  describe '#pool' do
+    let(:factory) { described_class.new }
+
+    it 'responds to serializers interface' do
+      pool = factory.pool(1)
+      expect(pool.load(pool.dump(42))).to be == 42
+    end
+
+    it 'types can be registered before the pool is created' do
+      factory.register_type(0x00, Symbol)
+      pool = factory.pool(1)
+      expect(pool.load(pool.dump(:foo))).to be == :foo
+    end
+
+    it 'types cannot be registered after the pool is created' do
+      pool = factory.pool(1)
+      factory.register_type(0x20, ::MyType)
+
+      expect do
+        pool.dump(MyType.new(1, 2))
+      end.to raise_error NoMethodError
+
+      payload = factory.dump(MyType.new(1, 2))
+      expect do
+        pool.load(payload)
+      end.to raise_error MessagePack::UnknownExtTypeError
+    end
+
+    it 'support symbolize_keys: true' do
+      pool = factory.pool(1, symbolize_keys: true)
+      expect(pool.load(pool.dump('foo' => 1))).to be == { foo: 1 }
+    end
+
+    it 'support freeze: true' do
+      pool = factory.pool(1, freeze: true)
+      expect(pool.load(pool.dump('foo'))).to be_frozen
+    end
+
+    it 'is thread safe' do
+      pool = factory.pool(1)
+
+      threads = 10.times.map do
+        Thread.new do
+          1_000.times do |i|
+            expect(pool.load(pool.dump(i))).to be == i
+          end
+        end
+      end
+      threads.each(&:join)
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -1,3 +1,4 @@
+require "set"
 
 if ENV['SIMPLE_COV']
   require 'simplecov'
@@ -14,11 +15,16 @@ if ENV['GC_STRESS']
 end
 
 require 'msgpack'
+require "msgpack/bigint"
 
 if GC.respond_to?(:verify_compaction_references)
   # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
   # move objects around, helping to find object movement bugs.
-  GC.verify_compaction_references(double_heap: true, toward: :empty)
+  begin
+    GC.verify_compaction_references(double_heap: true, toward: :empty)
+  rescue NotImplementedError
+    # Some platforms don't support compaction
+  end
 end
 
 if GC.respond_to?(:auto_compact=)
@@ -27,6 +33,8 @@ end
 
 IS_JRUBY = RUBY_ENGINE == 'jruby'
 
+IS_TRUFFLERUBY = RUBY_ENGINE == 'truffleruby'
+
 # checking if Hash#[]= (rb_hash_aset) dedupes string keys
 def automatic_string_keys_deduplication?
   h = {}

data/spec/timestamp_spec.rb

@@ -87,14 +87,14 @@ describe MessagePack::Timestamp do
 
     let(:time96_min) { Time.at(-2**63).utc }
     it 'is serialized into timestamp96' do
-      skip if IS_JRUBY # JRuby cannot handle numerics larger than long
+      skip if IS_JRUBY || IS_TRUFFLERUBY # JRuby and TruffleRuby both use underlying Java time classes that do not support |year| >= 1 billion
       expect(factory.pack(time96_min).size).to be 15
       expect(factory.unpack(factory.pack(time96_min)).utc).to eq(time96_min)
     end
 
     let(:time96_max) { Time.at(2**63 - 1).utc }
     it 'is serialized into timestamp96' do
-      skip if IS_JRUBY # JRuby cannot handle numerics larger than long
+      skip if IS_JRUBY || IS_TRUFFLERUBY # JRuby and TruffleRuby both use underlying Java time classes that do not support |year| >= 1 billion
       expect(factory.pack(time96_max).size).to be 15
       expect(factory.unpack(factory.pack(time96_max)).utc).to eq(time96_max)
     end

data/spec/unpacker_spec.rb

@@ -707,6 +707,18 @@ describe MessagePack::Unpacker do
         described_class.new(:freeze => true)
       end
 
+      if (-"test").equal?(-"test") # RUBY_VERSION >= "2.5"
+        it 'dedups strings' do
+          interned_str = -"test"
+          roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
+          expect(roundtrip).to be interned_str
+
+          interned_str = -""
+          roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
+          expect(roundtrip).to be interned_str
+        end
+      end
+
       it 'can freeze objects when using .unpack' do
         parsed_struct = MessagePack.unpack(buffer, freeze: true)
         parsed_struct.should == struct

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: msgpack
 version: !ruby/object:Gem::Version
-  version: 1.4.5
+  version: 1.5.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-15 00:00:00.000000000 Z
+date: 2022-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -96,6 +96,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.0
 description: MessagePack is a binary-based efficient object serialization library.
   It enables to exchange structured objects between many languages like JSON. But
   unlike JSON, it is very fast and small.
@@ -117,16 +131,7 @@ files:
 - README.md
 - Rakefile
 - appveyor.yml
-- bench/pack.rb
-- bench/pack_log.rb
-- bench/pack_log_long.rb
-- bench/pack_symbols.rb
-- bench/run.sh
-- bench/run_long.sh
-- bench/run_symbols.sh
-- bench/unpack.rb
-- bench/unpack_log.rb
-- bench/unpack_log_long.rb
+- bench/bench.rb
 - doclib/msgpack.rb
 - doclib/msgpack/buffer.rb
 - doclib/msgpack/core_ext.rb
@@ -176,6 +181,7 @@ files:
 - ext/msgpack/unpacker_ext_registry.c
 - ext/msgpack/unpacker_ext_registry.h
 - lib/msgpack.rb
+- lib/msgpack/bigint.rb
 - lib/msgpack/core_ext.rb
 - lib/msgpack/factory.rb
 - lib/msgpack/packer.rb
@@ -186,6 +192,7 @@ files:
 - lib/msgpack/version.rb
 - msgpack.gemspec
 - msgpack.org.md
+- spec/bigint_spec.rb
 - spec/cases.json
 - spec/cases.msg
 - spec/cases_compact.msg
@@ -229,7 +236,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: MessagePack, a binary-based efficient data interchange format.

data/bench/pack.rb

@@ -1,23 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data = { 'hello' => 'world', 'nested' => ['structure', {value: 42}] }
-data_sym = { hello: 'world', nested: ['structure', {value: 42}] }
-
-data = MessagePack.pack(:hello => 'world', :nested => ['structure', {:value => 42}])
-
-Viiite.bench do |b|
-  b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
-    b.report(:strings) do
-      runs.times do
-        MessagePack.pack(data)
-      end
-    end
-
-    b.report(:symbols) do
-      runs.times do
-        MessagePack.pack(data_sym)
-      end
-    end
-  end
-end

data/bench/pack_log.rb

@@ -1,33 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data_plain = { 'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"' }
-data_structure = {
-  'remote_host' => '127.0.0.1',
-  'remote_user' => '-',
-  'date' => '10/Oct/2000:13:55:36 -0700',
-  'request' => 'GET /apache_pb.gif HTTP/1.0',
-  'method' => 'GET',
-  'path' => '/apache_pb.gif',
-  'protocol' => 'HTTP/1.0',
-  'status' => 200,
-  'bytes' => 2326,
-  'referer' => 'http://www.example.com/start.html',
-  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
-}
-
-Viiite.bench do |b|
-  b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
-    b.report(:plain) do
-      runs.times do
-        MessagePack.pack(data_plain)
-      end
-    end
-
-    b.report(:structure) do
-      runs.times do
-        MessagePack.pack(data_structure)
-      end
-    end
-  end
-end