msgpack 1.4.2 → 1.6.0

data/spec/packer_spec.rb

@@ -488,6 +488,24 @@ describe MessagePack::Packer do
       it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
     end
 
+    shared_examples_for 'extension subclasses core type' do |klass|
+      before { stub_const('Value', Class.new(klass)) }
+      let(:object) { Value.new }
+      subject { packer.pack(object).to_s }
+
+      it "defaults to #{klass.name} packer if no extension is present" do
+        expect(subject).to eq(MessagePack.dump(klass.new))
+      end
+
+      it "uses core type extension for #{klass.name}" do
+        packer.register_type(0x01, Value, ->(_) { 'value_msgpacked' })
+        expect(subject).to eq("\xC7\x0F\x01value_msgpacked")
+      end
+    end
+    it_behaves_like 'extension subclasses core type', Hash
+    it_behaves_like 'extension subclasses core type', Array
+    it_behaves_like 'extension subclasses core type', String
+
     context 'when registering a type for symbols' do
       before { packer.register_type(0x00, ::Symbol, :to_msgpack_ext) }
 

data/spec/spec_helper.rb

@@ -1,3 +1,5 @@
+require "set"
+require "objspace"
 
 if ENV['SIMPLE_COV']
   require 'simplecov'
@@ -14,11 +16,26 @@ if ENV['GC_STRESS']
 end
 
 require 'msgpack'
+require "msgpack/bigint"
 
-def java?
-  /java/ =~ RUBY_PLATFORM
+if GC.respond_to?(:verify_compaction_references)
+  # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
+  # move objects around, helping to find object movement bugs.
+  begin
+    GC.verify_compaction_references(double_heap: true, toward: :empty)
+  rescue NotImplementedError
+    # Some platforms don't support compaction
+  end
+end
+
+if GC.respond_to?(:auto_compact=)
+  GC.auto_compact = true
 end
 
+IS_JRUBY = RUBY_ENGINE == 'jruby'
+
+IS_TRUFFLERUBY = RUBY_ENGINE == 'truffleruby'
+
 # checking if Hash#[]= (rb_hash_aset) dedupes string keys
 def automatic_string_keys_deduplication?
   h = {}
@@ -36,7 +53,7 @@ def string_deduplication?
   (-r1).equal?(-r2)
 end
 
-if java?
+if IS_JRUBY
   RSpec.configure do |c|
     c.treat_symbols_as_metadata_keys_with_true_values = true
     c.filter_run_excluding :encodings => !(defined? Encoding)

data/spec/timestamp_spec.rb

@@ -60,6 +60,44 @@ describe MessagePack::Timestamp do
     it 'runs correctly (regression)' do
       expect(factory.unpack(factory.pack(Time.utc(2200)))).to eq(Time.utc(2200))
     end
+
+    let(:time32_max) { Time.new(2106, 2, 7, 6, 28, 15, "+00:00") }
+    it 'is serialized into timestamp32' do
+      expect(factory.pack(time32_max).size).to be 6
+      expect(factory.unpack(factory.pack(time32_max)).utc).to eq(time32_max)
+    end
+
+    let(:time64_min) { Time.new(2106, 2, 7, 6, 28, 16, "+00:00") }
+    it 'is serialized into timestamp64' do
+      expect(factory.pack(time64_min).size).to be 10
+      expect(factory.unpack(factory.pack(time64_min)).utc).to eq(time64_min)
+    end
+
+    let(:time64_max) { Time.at(Time.new(2514, 5, 30, 1, 53, 3, "+00:00").to_i, 999999999 / 1000.0r).utc } # TODO: use Time.at(sec, nsec, :nsec) when removing Ruby 2.4 from the list
+    it 'is serialized into timestamp64' do
+      expect(factory.pack(time64_max).size).to be 10
+      expect(factory.unpack(factory.pack(time64_max)).utc).to eq(time64_max)
+    end
+
+    let(:time96_positive_min) { Time.new(2514, 5, 30, 1, 53, 4, "+00:00") }
+    it 'is serialized into timestamp96' do
+      expect(factory.pack(time96_positive_min).size).to be 15
+      expect(factory.unpack(factory.pack(time96_positive_min)).utc).to eq(time96_positive_min)
+    end
+
+    let(:time96_min) { Time.at(-2**63).utc }
+    it 'is serialized into timestamp96' do
+      skip if IS_JRUBY || IS_TRUFFLERUBY # JRuby and TruffleRuby both use underlying Java time classes that do not support |year| >= 1 billion
+      expect(factory.pack(time96_min).size).to be 15
+      expect(factory.unpack(factory.pack(time96_min)).utc).to eq(time96_min)
+    end
+
+    let(:time96_max) { Time.at(2**63 - 1).utc }
+    it 'is serialized into timestamp96' do
+      skip if IS_JRUBY || IS_TRUFFLERUBY # JRuby and TruffleRuby both use underlying Java time classes that do not support |year| >= 1 billion
+      expect(factory.pack(time96_max).size).to be 15
+      expect(factory.unpack(factory.pack(time96_max)).utc).to eq(time96_max)
+    end
   end
 
   describe 'register_type with MessagePack::Timestamp' do

data/spec/unpacker_spec.rb

@@ -1,5 +1,3 @@
-# encoding: ascii-8bit
-
 require 'stringio'
 require 'tempfile'
 require 'zlib'
@@ -36,6 +34,15 @@ describe MessagePack::Unpacker do
       unpacker.each { |obj| hashes = obj }
       expect(hashes[0].keys.first).to equal(hashes[1].keys.first)
     end
+
+    it 'ensure strings are not deduplicated' do
+      sample_data = ["foo"]
+      sample_packed = MessagePack.pack(sample_data).force_encoding('ASCII-8BIT')
+      unpacker.feed(sample_packed)
+      ary = nil
+      unpacker.each { |obj| ary = obj }
+      expect(ary.first.frozen?).to eq(false)
+    end
   end
 
   it 'gets IO or object which has #read to read data from it' do
@@ -293,6 +300,21 @@ describe MessagePack::Unpacker do
     MessagePack.unpack(MessagePack.pack(symbolized_hash), :symbolize_keys => true).should == symbolized_hash
   end
 
+  it 'MessagePack.unpack symbolize_keys preserve encoding' do
+    hash = { :ascii => 1, :utf8_é => 2}
+    loaded_hash = MessagePack.load(MessagePack.pack(hash), :symbolize_keys => true)
+
+    hash.keys[0].encoding.should == Encoding::US_ASCII # Ruby coerce symbols to US-ASCII when possible.
+    loaded_hash.keys[0].should == hash.keys[0]
+    loaded_hash.keys[0].encoding.should == hash.keys[0].encoding
+
+    hash.keys[1].encoding.should == Encoding::UTF_8
+    loaded_hash.keys[1].should == hash.keys[1]
+    loaded_hash.keys[1].encoding.should == hash.keys[1].encoding
+
+    MessagePack.unpack(MessagePack.pack(hash), :symbolize_keys => true).should == hash
+  end
+
   it 'Unpacker#unpack symbolize_keys' do
     unpacker = MessagePack::Unpacker.new(:symbolize_keys => true)
     symbolized_hash = {:a => 'b', :c => 'd'}
@@ -635,13 +657,23 @@ describe MessagePack::Unpacker do
       MessagePack.unpack(MessagePack.pack(array)).size.should == 10_000
     end
 
-    it 'preserve string encoding (issue #200)' do
+    it 'preserves string encoding (issue #200)' do
       string = 'a'.force_encoding(Encoding::UTF_8)
       MessagePack.unpack(MessagePack.pack(string)).encoding.should == string.encoding
 
       string *= 256
       MessagePack.unpack(MessagePack.pack(string)).encoding.should == string.encoding
     end
+
+    it 'returns correct size for array16 (issue #127)' do
+      unpacker.feed("\xdc\x00\x01\x01")
+      unpacker.read_array_header.should == 1
+    end
+
+    it 'returns correct size for map16 (issue #127)' do
+      unpacker.feed("\xde\x00\x02\x01\x02\x03\x04")
+      unpacker.read_map_header.should == 2
+    end
   end
 
   context 'extensions' do
@@ -685,6 +717,18 @@ describe MessagePack::Unpacker do
         described_class.new(:freeze => true)
       end
 
+      if (-"test").equal?(-"test") # RUBY_VERSION >= "2.5"
+        it 'dedups strings' do
+          interned_str = -"test"
+          roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
+          expect(roundtrip).to be interned_str
+
+          interned_str = -""
+          roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
+          expect(roundtrip).to be interned_str
+        end
+      end
+
       it 'can freeze objects when using .unpack' do
         parsed_struct = MessagePack.unpack(buffer, freeze: true)
         parsed_struct.should == struct
@@ -756,7 +800,13 @@ describe MessagePack::Unpacker do
 
     context 'binary encoding', :encodings do
       let :buffer do
-        MessagePack.pack({'hello' => 'world', 'nested' => ['object', {'structure' => true}]})
+        MessagePack.pack({
+          'hello'.b => 'world'.b,
+          'nested'.b => [
+            'object'.b,
+            {'structure'.b => true},
+          ]
+        })
       end
 
       let :unpacker do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: msgpack
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.6.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-01 00:00:00.000000000 Z
+date: 2022-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -46,14 +46,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.9
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.9
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -96,6 +96,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.0
 description: MessagePack is a binary-based efficient object serialization library.
   It enables to exchange structured objects between many languages like JSON. But
   unlike JSON, it is very fast and small.
@@ -108,25 +122,17 @@ extensions:
 - ext/msgpack/extconf.rb
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yaml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - ChangeLog
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - appveyor.yml
-- bench/pack.rb
-- bench/pack_log.rb
-- bench/pack_log_long.rb
-- bench/pack_symbols.rb
-- bench/run.sh
-- bench/run_long.sh
-- bench/run_symbols.sh
-- bench/unpack.rb
-- bench/unpack_log.rb
-- bench/unpack_log_long.rb
+- bench/bench.rb
+- bin/console
 - doclib/msgpack.rb
 - doclib/msgpack/buffer.rb
 - doclib/msgpack/core_ext.rb
@@ -176,6 +182,7 @@ files:
 - ext/msgpack/unpacker_ext_registry.c
 - ext/msgpack/unpacker_ext_registry.h
 - lib/msgpack.rb
+- lib/msgpack/bigint.rb
 - lib/msgpack/core_ext.rb
 - lib/msgpack/factory.rb
 - lib/msgpack/packer.rb
@@ -186,6 +193,7 @@ files:
 - lib/msgpack/version.rb
 - msgpack.gemspec
 - msgpack.org.md
+- spec/bigint_spec.rb
 - spec/cases.json
 - spec/cases.msg
 - spec/cases_compact.msg
@@ -229,32 +237,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: MessagePack, a binary-based efficient data interchange format.
-test_files:
-- spec/cases.json
-- spec/cases.msg
-- spec/cases_compact.msg
-- spec/cases_spec.rb
-- spec/cruby/buffer_io_spec.rb
-- spec/cruby/buffer_packer.rb
-- spec/cruby/buffer_spec.rb
-- spec/cruby/buffer_unpacker.rb
-- spec/cruby/unpacker_spec.rb
-- spec/ext_value_spec.rb
-- spec/exttypes.rb
-- spec/factory_spec.rb
-- spec/format_spec.rb
-- spec/jruby/benchmarks/shootout_bm.rb
-- spec/jruby/benchmarks/symbolize_keys_bm.rb
-- spec/jruby/unpacker_spec.rb
-- spec/msgpack_spec.rb
-- spec/pack_spec.rb
-- spec/packer_spec.rb
-- spec/random_compat.rb
-- spec/spec_helper.rb
-- spec/timestamp_spec.rb
-- spec/unpack_spec.rb
-- spec/unpacker_spec.rb
+test_files: []

data/.travis.yml

@@ -1,39 +0,0 @@
-language: ruby
-
-branches:
-  only:
-    - master
-
-gemfile:
-  - Gemfile
-
-before_install:
-  # This is only for Ruby 2.5.0, Bundler 1.16.1 and rubygems 2.7.3
-  # See https://github.com/travis-ci/travis-ci/issues/8969
-  - "[ \"x2.7.3\" = \"x\"$(gem --version) ] && gem update --system --no-document || echo \"no need to update rubygem\""
-
-# http://rubies.travis-ci.org/
-matrix:
-  include:
-    - rvm: 2.4.5
-      os: linux
-    - rvm: 2.5.8
-      os: linux
-    - rvm: 2.6.6
-      os: linux
-    - rvm: 2.6.6
-      os: osx
-    - rvm: 2.7.1
-      os: linux
-    - rvm: ruby-head
-      os: linux
-    - rvm: jruby-head
-      os: linux
-    - rvm: jruby-19mode
-      os: linux
-  allow_failures:
-    - rvm: 2.6.1
-      os: osx
-    - rvm: ruby-head
-    - rvm: jruby-head
-    - rvm: jruby-19mode

data/bench/pack.rb

@@ -1,23 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data = { 'hello' => 'world', 'nested' => ['structure', {value: 42}] }
-data_sym = { hello: 'world', nested: ['structure', {value: 42}] }
-
-data = MessagePack.pack(:hello => 'world', :nested => ['structure', {:value => 42}])
-
-Viiite.bench do |b|
-  b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
-    b.report(:strings) do
-      runs.times do
-        MessagePack.pack(data)
-      end
-    end
-
-    b.report(:symbols) do
-      runs.times do
-        MessagePack.pack(data_sym)
-      end
-    end
-  end
-end

data/bench/pack_log.rb

@@ -1,33 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data_plain = { 'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"' }
-data_structure = {
-  'remote_host' => '127.0.0.1',
-  'remote_user' => '-',
-  'date' => '10/Oct/2000:13:55:36 -0700',
-  'request' => 'GET /apache_pb.gif HTTP/1.0',
-  'method' => 'GET',
-  'path' => '/apache_pb.gif',
-  'protocol' => 'HTTP/1.0',
-  'status' => 200,
-  'bytes' => 2326,
-  'referer' => 'http://www.example.com/start.html',
-  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
-}
-
-Viiite.bench do |b|
-  b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
-    b.report(:plain) do
-      runs.times do
-        MessagePack.pack(data_plain)
-      end
-    end
-
-    b.report(:structure) do
-      runs.times do
-        MessagePack.pack(data_structure)
-      end
-    end
-  end
-end

data/bench/pack_log_long.rb

@@ -1,65 +0,0 @@
-# viiite report --regroup bench,threads bench/pack_log_long.rb
-
-require 'viiite'
-require 'msgpack'
-
-data_plain = { 'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"' }
-data_structure = {
-  'remote_host' => '127.0.0.1',
-  'remote_user' => '-',
-  'date' => '10/Oct/2000:13:55:36 -0700',
-  'request' => 'GET /apache_pb.gif HTTP/1.0',
-  'method' => 'GET',
-  'path' => '/apache_pb.gif',
-  'protocol' => 'HTTP/1.0',
-  'status' => 200,
-  'bytes' => 2326,
-  'referer' => 'http://www.example.com/start.html',
-  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
-}
-
-seconds = 3600 # 1 hour
-
-Viiite.bench do |b|
-  b.range_over([1, 2, 4, 8, 16], :threads) do |threads|
-    b.report(:plain) do
-      ths = []
-      end_at = Time.now + seconds
-      threads.times do
-        t = Thread.new do
-          packs = 0
-          while Time.now < end_at
-            10000.times do
-              MessagePack.pack(data_plain)
-            end
-            packs += 10000
-          end
-          packs
-        end
-        ths.push t
-      end
-      sum = ths.reduce(0){|r,t| r + t.value }
-      puts "MessagePack.pack, plain, #{threads} threads: #{sum} times, #{sum / seconds} times/second."
-    end
-
-    b.report(:structure) do
-      ths = []
-      end_at = Time.now + seconds
-      threads.times do
-        t = Thread.new do
-          packs = 0
-          while Time.now < end_at
-            10000.times do
-              MessagePack.pack(data_structure)
-            end
-            packs += 10000
-          end
-          packs
-        end
-        ths.push t
-      end
-      sum = ths.reduce(0){|r,t| r + t.value }
-      puts "MessagePack.pack, structured, #{threads} threads: #{sum} times, #{sum / seconds} times/second."
-    end
-  end
-end

data/bench/pack_symbols.rb

@@ -1,28 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data = :symbol
-
-Viiite.bench do |b|
-  b.variation_point :branch, `git rev-parse --abbrev-ref HEAD`
-
-  b.range_over([:symbol, :none], :reg_type) do |reg_type|
-    packer = MessagePack::Packer.new
-    packer.register_type(0x00, Symbol, :to_msgpack_ext) if reg_type == :symbol
-
-    b.range_over([100_000, 1_000_000, 10_000_000], :count) do |count|
-      packer.clear
-      b.report(:multi_run) do
-        count.times do
-          packer.pack(data)
-        end
-      end
-
-      packer.clear
-      items_data = [].fill(data, 0, count)
-      b.report(:large_run) do
-        packer.pack(items_data)
-      end
-    end
-  end
-end

data/bench/run.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# prerequisites
-# $ rbenv shell 2.2.1 (or jruby-x.x.x or ...)
-# $ rake install
-
-echo "pack"
-viiite report --regroup bench,runs bench/pack.rb 
-echo "unpack"
-viiite report --regroup bench,runs bench/unpack.rb 
-echo "pack log"
-viiite report --regroup bench,runs bench/pack_log.rb 
-echo "unpack log"
-viiite report --regroup bench,runs bench/unpack_log.rb 

data/bench/run_long.sh

@@ -1,35 +0,0 @@
-#!/bin/sh
-
-# prerequisites
-# $ sudo apt-get install sysstat
-# $ rbenv shell 2.2.1 (or jruby-x.x.x or ...)
-# $ rake install
-
-# 60 * 600 : 60*60 * 5[threads] * 2[bench]
-
-ruby -v
-
-echo "pack log long"
-viiite report --regroup bench,threads bench/pack_log_long.rb &
-sar -o pack_log_long.sar -r 60 600 > /dev/null 2>&1 &
-
-declare -i i=0
-while [ $i -lt 600 ]; do
-    ps auxww | grep ruby | grep -v grep | awk '{print $5,$6;}' >> pack_log_long.mem.txt
-    i=i+1
-    sleep 60
-done
-
-sleep 120 # cool down
-
-echo "unpack log long"
-viiite report --regroup bench,threads bench/unpack_log_long.rb &
-sar -o unpack_log_long.sar -r 60 600 > /dev/null 2>&1 &
-
-i=0
-while [ $i -lt 600 ]; do
-    ps auxww | grep ruby | grep -v grep | awk '{print $5,$6;}' >> pack_log_long.mem.txt
-    i=i+1
-    sleep 60
-done
-

data/bench/run_symbols.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-# so master and this branch have the benchmark file in any case
-cp bench/pack_symbols.rb bench/pack_symbols_tmp.rb
-
-benchmark=""
-current_branch=`git rev-parse --abbrev-ref HEAD`
-
-for branch in master $current_branch; do
-    echo "Testing branch $branch"
-    git checkout $branch
-
-    echo "Installing gem..."
-    rake install
-
-    echo "Running benchmark..."
-    if [ "$benchmark" ]; then
-        benchmark+=$'\n'
-    fi
-    benchmark+=$(viiite run bench/pack_symbols_tmp.rb)
-    echo
-done
-
-rm bench/pack_symbols_tmp.rb
-
-echo "$benchmark" | viiite report --regroup bench,reg_type,count,branch

data/bench/unpack.rb

@@ -1,21 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data = MessagePack.pack(:hello => 'world', :nested => ['structure', {:value => 42}])
-
-Viiite.bench do |b|
-  b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
-    b.report(:strings) do
-      runs.times do
-        MessagePack.unpack(data)
-      end
-    end
-
-    b.report(:symbols) do
-      options = {:symbolize_keys => true}
-      runs.times do
-        MessagePack.unpack(data, options)
-      end
-    end
-  end
-end

data/bench/unpack_log.rb

@@ -1,34 +0,0 @@
-require 'viiite'
-require 'msgpack'
-
-data_plain = MessagePack.pack({ 'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"' })
-
-data_structure = MessagePack.pack({
-  'remote_host' => '127.0.0.1',
-  'remote_user' => '-',
-  'date' => '10/Oct/2000:13:55:36 -0700',
-  'request' => 'GET /apache_pb.gif HTTP/1.0',
-  'method' => 'GET',
-  'path' => '/apache_pb.gif',
-  'protocol' => 'HTTP/1.0',
-  'status' => 200,
-  'bytes' => 2326,
-  'referer' => 'http://www.example.com/start.html',
-  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
-})
-
-Viiite.bench do |b|
-  b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
-    b.report(:plain) do
-      runs.times do
-        MessagePack.unpack(data_plain)
-      end
-    end
-
-    b.report(:structure) do
-      runs.times do
-        MessagePack.unpack(data_structure)
-      end
-    end
-  end
-end