msgpack 1.2.9 → 1.4.4

data/ext/msgpack/unpacker_class.c

@@ -77,9 +77,6 @@ VALUE MessagePack_Unpacker_initialize(int argc, VALUE* argv, VALUE self)
         VALUE v = argv[0];
         if(rb_type(v) == T_HASH) {
             options = v;
-            if(rb_type(options) != T_HASH) {
-                rb_raise(rb_eArgError, "expected Hash but found %s.", rb_obj_classname(options));
-            }
         } else {
             io = v;
         }
@@ -108,6 +105,9 @@ VALUE MessagePack_Unpacker_initialize(int argc, VALUE* argv, VALUE self)
         v = rb_hash_aref(options, ID2SYM(rb_intern("symbolize_keys")));
         msgpack_unpacker_set_symbolized_keys(uk, RTEST(v));
 
+        v = rb_hash_aref(options, ID2SYM(rb_intern("freeze")));
+        msgpack_unpacker_set_freeze(uk, RTEST(v));
+
         v = rb_hash_aref(options, ID2SYM(rb_intern("allow_unknown_ext")));
         msgpack_unpacker_set_allow_unknown_ext(uk, RTEST(v));
     }
@@ -121,6 +121,12 @@ static VALUE Unpacker_symbolized_keys_p(VALUE self)
     return uk->symbolize_keys ? Qtrue : Qfalse;
 }
 
+static VALUE Unpacker_freeze_p(VALUE self)
+{
+    UNPACKER(self, uk);
+    return uk->freeze ? Qtrue : Qfalse;
+}
+
 static VALUE Unpacker_allow_unknown_ext_p(VALUE self)
 {
     UNPACKER(self, uk);
@@ -256,6 +262,17 @@ static VALUE Unpacker_feed(VALUE self, VALUE data)
     return self;
 }
 
+static VALUE Unpacker_feed_reference(VALUE self, VALUE data)
+{
+    UNPACKER(self, uk);
+
+    StringValue(data);
+
+    msgpack_buffer_append_string_reference(UNPACKER_BUFFER_(uk), data);
+
+    return self;
+}
+
 static VALUE Unpacker_each_impl(VALUE self)
 {
     UNPACKER(self, uk);
@@ -312,8 +329,7 @@ static VALUE Unpacker_feed_each(VALUE self, VALUE data)
     }
 #endif
 
-    // TODO optimize
-    Unpacker_feed(self, data);
+    Unpacker_feed_reference(self, data);
     return Unpacker_each(self);
 }
 
@@ -353,12 +369,7 @@ static VALUE Unpacker_register_type(int argc, VALUE* argv, VALUE self)
     case 1:
         /* register_type(0x7f) {|data| block... } */
         rb_need_block();
-#ifdef HAVE_RB_BLOCK_LAMBDA
         proc = rb_block_lambda();
-#else
-        /* MRI 1.8 */
-        proc = rb_block_proc();
-#endif
         arg = proc;
         ext_module = Qnil;
         break;
@@ -386,9 +397,6 @@ static VALUE Unpacker_full_unpack(VALUE self)
 {
     UNPACKER(self, uk);
 
-    /* prefer reference than copying; see MessagePack_Unpacker_module_init */
-    msgpack_buffer_set_write_reference_threshold(UNPACKER_BUFFER_(uk), 0);
-
     int r = msgpack_unpacker_read(uk, 0);
     if(r < 0) {
         raise_unpacker_error(r);
@@ -434,6 +442,7 @@ void MessagePack_Unpacker_module_init(VALUE mMessagePack)
 
     rb_define_method(cMessagePack_Unpacker, "initialize", MessagePack_Unpacker_initialize, -1);
     rb_define_method(cMessagePack_Unpacker, "symbolize_keys?", Unpacker_symbolized_keys_p, 0);
+    rb_define_method(cMessagePack_Unpacker, "freeze?", Unpacker_freeze_p, 0);
     rb_define_method(cMessagePack_Unpacker, "allow_unknown_ext?", Unpacker_allow_unknown_ext_p, 0);
     rb_define_method(cMessagePack_Unpacker, "buffer", Unpacker_buffer, 0);
     rb_define_method(cMessagePack_Unpacker, "read", Unpacker_read, 0);
@@ -444,6 +453,7 @@ void MessagePack_Unpacker_module_init(VALUE mMessagePack)
     rb_define_method(cMessagePack_Unpacker, "read_map_header", Unpacker_read_map_header, 0);
     //rb_define_method(cMessagePack_Unpacker, "peek_next_type", Unpacker_peek_next_type, 0);  // TODO
     rb_define_method(cMessagePack_Unpacker, "feed", Unpacker_feed, 1);
+    rb_define_method(cMessagePack_Unpacker, "feed_reference", Unpacker_feed_reference, 1);
     rb_define_method(cMessagePack_Unpacker, "each", Unpacker_each, 0);
     rb_define_method(cMessagePack_Unpacker, "feed_each", Unpacker_feed_each, 1);
     rb_define_method(cMessagePack_Unpacker, "reset", Unpacker_reset, 0);

data/lib/msgpack/symbol.rb

@@ -1,9 +1,19 @@
 class Symbol
-  def to_msgpack_ext
-    [to_s].pack('A*')
+  # to_msgpack_ext is supposed to return a binary string.
+  # The canonical way to do it for symbols would be:
+  #  [to_s].pack('A*')
+  # However in this instance we can take a shortcut
+  if method_defined?(:name)
+    alias_method :to_msgpack_ext, :name
+  else
+    alias_method :to_msgpack_ext, :to_s
   end
 
   def self.from_msgpack_ext(data)
-    data.unpack('A*').first.to_sym
+    # from_msgpack_ext is supposed to parse a binary string.
+    # The canonical way to do it for symbols would be:
+    #  data.unpack1('A*').to_sym
+    # However in this instance we can take a shortcut
+    data.to_sym
   end
-end
+end

data/lib/msgpack/time.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# MessagePack extention packer and unpacker for built-in Time class
+module MessagePack
+  module Time
+    # 3-arg Time.at is available Ruby >= 2.5
+    TIME_AT_3_AVAILABLE = begin
+                            !!::Time.at(0, 0, :nanosecond)
+                          rescue ArgumentError
+                            false
+                          end
+
+    Unpacker = if TIME_AT_3_AVAILABLE
+                 lambda do |payload|
+                   tv = MessagePack::Timestamp.from_msgpack_ext(payload)
+                   ::Time.at(tv.sec, tv.nsec, :nanosecond)
+                 end
+               else
+                 lambda do |payload|
+                   tv = MessagePack::Timestamp.from_msgpack_ext(payload)
+                   ::Time.at(tv.sec, tv.nsec / 1000.0r)
+                 end
+               end
+
+    Packer = lambda { |time|
+      MessagePack::Timestamp.to_msgpack_ext(time.tv_sec, time.tv_nsec)
+    }
+  end
+end

data/lib/msgpack/timestamp.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module MessagePack
+  class Timestamp # a.k.a. "TimeSpec"
+    # Because the byte-order of MessagePack is big-endian in,
+    # pack() and unpack() specifies ">".
+    # See https://docs.ruby-lang.org/en/trunk/Array.html#method-i-pack for details.
+
+    # The timestamp extension type defined in the MessagePack spec.
+    # See https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type for details.
+    TYPE = -1
+
+    TIMESTAMP32_MAX_SEC = (1 << 32) - 1
+    TIMESTAMP64_MAX_SEC = (1 << 34) - 1
+
+    # @return [Integer]
+    attr_reader :sec
+
+    # @return [Integer]
+    attr_reader :nsec
+
+    # @param [Integer] sec
+    # @param [Integer] nsec
+    def initialize(sec, nsec)
+      @sec = sec
+      @nsec = nsec
+    end
+
+    def self.from_msgpack_ext(data)
+      case data.length
+      when 4
+        # timestamp32 (sec: uint32be)
+        sec, = data.unpack('L>')
+        new(sec, 0)
+      when 8
+        # timestamp64 (nsec: uint30be, sec: uint34be)
+        n, s = data.unpack('L>2')
+        sec = ((n & 0b11) << 32) | s
+        nsec = n >> 2
+        new(sec, nsec)
+      when 12
+        # timestam96 (nsec: uint32be, sec: int64be)
+        nsec, sec = data.unpack('L>q>')
+        new(sec, nsec)
+      else
+        raise MalformedFormatError, "Invalid timestamp data size: #{data.length}"
+      end
+    end
+
+    def self.to_msgpack_ext(sec, nsec)
+      if sec >= 0 && nsec >= 0 && sec <= TIMESTAMP64_MAX_SEC
+        if nsec === 0 && sec <= TIMESTAMP32_MAX_SEC
+          # timestamp32 = (sec: uint32be)
+          [sec].pack('L>')
+        else
+          # timestamp64 (nsec: uint30be, sec: uint34be)
+          nsec30 = nsec << 2
+          sec_high2 = sec >> 32 # high 2 bits (`x & 0b11` is redandunt)
+          sec_low32 = sec & 0xffffffff # low 32 bits
+          [nsec30 | sec_high2, sec_low32].pack('L>2')
+        end
+      else
+        # timestamp96 (nsec: uint32be, sec: int64be)
+        [nsec, sec].pack('L>q>')
+      end
+    end
+
+    def to_msgpack_ext
+      self.class.to_msgpack_ext(sec, nsec)
+    end
+
+    def ==(other)
+      other.class == self.class && sec == other.sec && nsec == other.nsec
+    end
+  end
+end

data/lib/msgpack/version.rb

@@ -1,9 +1,6 @@
 module MessagePack
-  VERSION = "1.2.9"
-
-  # NOTE for msgpack-ruby maintainer:
-  # Check these things to release new binaryes for new Ruby versions (especially for Windows):
-  # * versions/supports of rake-compiler & rake-compiler-dock
-  # * update RUBY_CC_VERSION in Rakefile
-  # * check Ruby dependency of released mswin gem details
+  VERSION = "1.4.4"
+  # Note for maintainers:
+  #  Don't miss building/releasing the JRuby version (rake buld:java)
+  #  See "How to build -java rubygems" in README for more details.
 end

data/lib/msgpack.rb

@@ -1,15 +1,10 @@
 require "msgpack/version"
 
 if defined?(RUBY_ENGINE) && RUBY_ENGINE == "jruby" # This is same with `/java/ =~ RUBY_VERSION`
-  require "java"
   require "msgpack/msgpack.jar"
-  org.msgpack.jruby.MessagePackLibrary.new.load(JRuby.runtime, false)
+  JRuby::Util.load_ext("org.msgpack.jruby.MessagePackLibrary")
 else
-  begin
-    require "msgpack/#{RUBY_VERSION[/\d+.\d+/]}/msgpack"
-  rescue LoadError
-    require "msgpack/msgpack"
-  end
+  require "msgpack/msgpack"
 end
 
 require "msgpack/packer"
@@ -17,6 +12,8 @@ require "msgpack/unpacker"
 require "msgpack/factory"
 require "msgpack/symbol"
 require "msgpack/core_ext"
+require "msgpack/timestamp"
+require "msgpack/time"
 
 module MessagePack
   DefaultFactory = MessagePack::Factory.new
@@ -27,7 +24,7 @@ module MessagePack
 
     if src.is_a? String
       unpacker = DefaultFactory.unpacker param || DEFAULT_EMPTY_PARAMS
-      unpacker.feed src
+      unpacker.feed_reference src
     else
       unpacker = DefaultFactory.unpacker src, param || DEFAULT_EMPTY_PARAMS
     end

data/msgpack.gemspec

@@ -10,7 +10,6 @@ Gem::Specification.new do |s|
   s.email = ["frsyuki@gmail.com", "theo@iconara.net", "tagomoris@gmail.com"]
   s.license = "Apache 2.0"
   s.homepage = "http://msgpack.org/"
-  s.rubyforge_project = "msgpack"
   s.require_paths = ["lib"]
   if /java/ =~ RUBY_PLATFORM
     s.files = Dir['lib/**/*.rb', 'lib/**/*.jar']
@@ -19,15 +18,12 @@ Gem::Specification.new do |s|
     s.files = `git ls-files`.split("\n")
     s.extensions = ["ext/msgpack/extconf.rb"]
   end
-  s.test_files = `git ls-files -- {test,spec}/*`.split("\n")
+
+  s.required_ruby_version = ">= 2.4"
 
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'rake'
-  s.add_development_dependency 'rake-compiler', ['~> 1.0']
-  if /java/ !~ RUBY_PLATFORM
-    # NOTE: rake-compiler-dock SHOULD be updated for new Ruby versions
-    s.add_development_dependency 'rake-compiler-dock', ['~> 0.7.0']
-  end
+  s.add_development_dependency 'rake-compiler', ['>= 1.1.9']
   s.add_development_dependency 'rspec', ['~> 3.3']
   s.add_development_dependency 'yard'
   s.add_development_dependency 'json'

data/spec/factory_spec.rb

@@ -280,6 +280,23 @@ describe MessagePack::Factory do
       unpacker.feed(packed_symbol).unpack
     end
 
+    context 'using the optimized symbol unpacker' do
+      before do
+        skip if IS_JRUBY # JRuby implementation doesn't support the optimized symbols unpacker for now
+        subject.register_type(
+          0x00,
+          ::Symbol,
+          packer: :to_msgpack_ext,
+          unpacker: :from_msgpack_ext,
+          optimized_symbols_parsing: true,
+        )
+      end
+
+      it 'lets symbols survive a roundtrip' do
+        expect(symbol_after_roundtrip).to be :symbol
+      end
+    end
+
     context 'if no ext type is registered for symbols' do
       it 'converts symbols to string' do
         expect(symbol_after_roundtrip).to eq 'symbol'

data/spec/msgpack_spec.rb

@@ -115,7 +115,7 @@ describe MessagePack do
       expect { MessagePack.pack(self) }.to raise_error(NoMethodError, /^undefined method `to_msgpack'/)
     end
 
-    it 'rasies an error on #unpack with garbage' do
+    it 'raises an error on #unpack with garbage' do
       skip "but nothing was raised. why?"
       expect { MessagePack.unpack('asdka;sd') }.to raise_error(MessagePack::UnpackError)
     end

data/spec/packer_spec.rb

@@ -488,6 +488,24 @@ describe MessagePack::Packer do
       it { is_expected.to eq "\xC7\x0F\x01value_msgpacked" }
     end
 
+    shared_examples_for 'extension subclasses core type' do |klass|
+      before { stub_const('Value', Class.new(klass)) }
+      let(:object) { Value.new }
+      subject { packer.pack(object).to_s }
+
+      it "defaults to #{klass.name} packer if no extension is present" do
+        expect(subject).to eq(MessagePack.dump(klass.new))
+      end
+
+      it "uses core type extension for #{klass.name}" do
+        packer.register_type(0x01, Value, ->(_) { 'value_msgpacked' })
+        expect(subject).to eq("\xC7\x0F\x01value_msgpacked")
+      end
+    end
+    it_behaves_like 'extension subclasses core type', Hash
+    it_behaves_like 'extension subclasses core type', Array
+    it_behaves_like 'extension subclasses core type', String
+
     context 'when registering a type for symbols' do
       before { packer.register_type(0x00, ::Symbol, :to_msgpack_ext) }
 

data/spec/spec_helper.rb

@@ -15,10 +15,37 @@ end
 
 require 'msgpack'
 
+if GC.respond_to?(:verify_compaction_references)
+  # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
+  # move objects around, helping to find object movement bugs.
+  GC.verify_compaction_references(double_heap: true, toward: :empty)
+end
+
+if GC.respond_to?(:auto_compact=)
+  GC.auto_compact = true
+end
+
 def java?
   /java/ =~ RUBY_PLATFORM
 end
 
+# checking if Hash#[]= (rb_hash_aset) dedupes string keys
+def automatic_string_keys_deduplication?
+  h = {}
+  x = {}
+  r = rand.to_s
+  h[%W(#{r}).join('')] = :foo
+  x[%W(#{r}).join('')] = :foo
+
+  x.keys[0].equal?(h.keys[0])
+end
+
+def string_deduplication?
+  r1 = rand.to_s
+  r2 = r1.dup
+  (-r1).equal?(-r2)
+end
+
 if java?
   RSpec.configure do |c|
     c.treat_symbols_as_metadata_keys_with_true_values = true

data/spec/timestamp_spec.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+IS_JRUBY = Kernel.const_defined?(:JRUBY_VERSION)
+
+describe MessagePack::Timestamp do
+  describe 'malformed format' do
+    it do
+      expect do
+        MessagePack::Timestamp.from_msgpack_ext([0xd4, 0x00].pack("C*"))
+      end.to raise_error(MessagePack::MalformedFormatError)
+    end
+  end
+
+  describe 'register_type with Time' do
+    let(:factory) do
+      factory = MessagePack::Factory.new
+      factory.register_type(
+        MessagePack::Timestamp::TYPE,
+        Time,
+        packer: MessagePack::Time::Packer,
+        unpacker: MessagePack::Time::Unpacker
+      )
+      factory
+    end
+
+    let(:time) { Time.local(2019, 6, 17, 1, 2, 3, 123_456_789 / 1000.0) }
+    it 'serializes and deserializes Time' do
+      prefix_fixext8_with_type_id = [0xd7, -1].pack("c*")
+
+      packed = factory.pack(time)
+      expect(packed).to start_with(prefix_fixext8_with_type_id)
+      expect(packed.size).to eq(10)
+      unpacked = factory.unpack(packed)
+      expect(unpacked.to_i).to eq(time.to_i)
+      expect(unpacked.to_f).to eq(time.to_f)
+      # expect(unpacked).to eq(time) # we can't do it because of nsec (rational vs float?)
+    end
+
+    let(:time_without_nsec) { Time.local(2019, 6, 17, 1, 2, 3, 0) }
+    it 'serializes time without nanosec as fixext4' do
+      prefix_fixext4_with_type_id = [0xd6, -1].pack("c*")
+
+      packed = factory.pack(time_without_nsec)
+      expect(packed).to start_with(prefix_fixext4_with_type_id)
+      expect(packed.size).to eq(6)
+      unpacked = factory.unpack(packed)
+      expect(unpacked).to eq(time_without_nsec)
+    end
+
+    let(:time_after_2514) { Time.at(1 << 34) } # the max num of 34bit int means 2514-05-30 01:53:04 UTC
+    it 'serializes time after 2038 as ext8' do
+      prefix_ext8_with_12bytes_payload_and_type_id = [0xc7, 12, -1].pack("c*")
+
+      expect(time_after_2514.to_i).to be > 0xffffffff
+      packed = factory.pack(time_after_2514)
+      expect(packed).to start_with(prefix_ext8_with_12bytes_payload_and_type_id)
+      expect(packed.size).to eq(15)
+    end
+
+    it 'runs correctly (regression)' do
+      expect(factory.unpack(factory.pack(Time.utc(2200)))).to eq(Time.utc(2200))
+    end
+
+    let(:time32_max) { Time.new(2106, 2, 7, 6, 28, 15, "+00:00") }
+    it 'is serialized into timestamp32' do
+      expect(factory.pack(time32_max).size).to be 6
+      expect(factory.unpack(factory.pack(time32_max)).utc).to eq(time32_max)
+    end
+
+    let(:time64_min) { Time.new(2106, 2, 7, 6, 28, 16, "+00:00") }
+    it 'is serialized into timestamp64' do
+      expect(factory.pack(time64_min).size).to be 10
+      expect(factory.unpack(factory.pack(time64_min)).utc).to eq(time64_min)
+    end
+
+    let(:time64_max) { Time.at(Time.new(2514, 5, 30, 1, 53, 3, "+00:00").to_i, 999999999 / 1000.0r).utc } # TODO: use Time.at(sec, nsec, :nsec) when removing Ruby 2.4 from the list
+    it 'is serialized into timestamp64' do
+      expect(factory.pack(time64_max).size).to be 10
+      expect(factory.unpack(factory.pack(time64_max)).utc).to eq(time64_max)
+    end
+
+    let(:time96_positive_min) { Time.new(2514, 5, 30, 1, 53, 4, "+00:00") }
+    it 'is serialized into timestamp96' do
+      expect(factory.pack(time96_positive_min).size).to be 15
+      expect(factory.unpack(factory.pack(time96_positive_min)).utc).to eq(time96_positive_min)
+    end
+
+    let(:time96_min) { Time.at(-2**63).utc }
+    it 'is serialized into timestamp96' do
+      skip if IS_JRUBY # JRuby cannot handle numerics larger than long
+      expect(factory.pack(time96_min).size).to be 15
+      expect(factory.unpack(factory.pack(time96_min)).utc).to eq(time96_min)
+    end
+
+    let(:time96_max) { Time.at(2**63 - 1).utc }
+    it 'is serialized into timestamp96' do
+      skip if IS_JRUBY # JRuby cannot handle numerics larger than long
+      expect(factory.pack(time96_max).size).to be 15
+      expect(factory.unpack(factory.pack(time96_max)).utc).to eq(time96_max)
+    end
+  end
+
+  describe 'register_type with MessagePack::Timestamp' do
+    let(:factory) do
+      factory = MessagePack::Factory.new
+      factory.register_type(MessagePack::Timestamp::TYPE, MessagePack::Timestamp)
+      factory
+    end
+
+    let(:timestamp) { MessagePack::Timestamp.new(Time.now.tv_sec, 123_456_789) }
+    it 'serializes and deserializes MessagePack::Timestamp' do
+      packed = factory.pack(timestamp)
+      unpacked = factory.unpack(packed)
+      expect(unpacked).to eq(timestamp)
+    end
+  end
+
+  describe 'timestamp32' do
+    it 'handles [1, 0]' do
+      t = MessagePack::Timestamp.new(1, 0)
+
+      payload = t.to_msgpack_ext
+      unpacked = MessagePack::Timestamp.from_msgpack_ext(payload)
+
+      expect(unpacked).to eq(t)
+    end
+  end
+
+  describe 'timestamp64' do
+    it 'handles [1, 1]' do
+      t = MessagePack::Timestamp.new(1, 1)
+
+      payload = t.to_msgpack_ext
+      unpacked = MessagePack::Timestamp.from_msgpack_ext(payload)
+
+      expect(unpacked).to eq(t)
+    end
+  end
+
+  describe 'timestamp96' do
+    it 'handles [-1, 0]' do
+      t = MessagePack::Timestamp.new(-1, 0)
+
+      payload = t.to_msgpack_ext
+      unpacked = MessagePack::Timestamp.from_msgpack_ext(payload)
+
+      expect(unpacked).to eq(t)
+    end
+
+    it 'handles [-1, 999_999_999]' do
+      t = MessagePack::Timestamp.new(-1, 999_999_999)
+
+      payload = t.to_msgpack_ext
+      unpacked = MessagePack::Timestamp.from_msgpack_ext(payload)
+
+      expect(unpacked).to eq(t)
+    end
+  end
+end