msgpack 0.5.10-java

data/spec/jruby/benchmarks/shootout_bm.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+
+if RUBY_PLATFORM.include?('java')
+  # JRuby should use this library, MRI should use the standard gem
+  $: << File.expand_path('../../../lib', __FILE__)
+end
+
+require 'viiite'
+require 'msgpack'
+require 'json'
+require 'bson'
+
+if RUBY_PLATFORM.include?('java')
+  BSON_IMPL = BSON::BSON_JAVA
+else
+  BSON_IMPL = BSON::BSON_C
+end
+
+OBJECT_STRUCTURE = {'x' => ['y', 34, 2**30 + 3, 2.1223423423356, {'hello' => 'world', '5' => [63, 'asjdl']}]}
+ENCODED_MSGPACK = "\x81\xA1x\x95\xA1y\"\xCE@\x00\x00\x03\xCB@\x00\xFA\x8E\x9F9\xFA\xC1\x82\xA5hello\xA5world\xA15\x92?\xA5asjdl"
+ENCODED_BSON = "d\x00\x00\x00\x04x\x00\\\x00\x00\x00\x020\x00\x02\x00\x00\x00y\x00\x101\x00\"\x00\x00\x00\x102\x00\x03\x00\x00@\x013\x00\xC1\xFA9\x9F\x8E\xFA\x00@\x034\x002\x00\x00\x00\x02hello\x00\x06\x00\x00\x00world\x00\x045\x00\x19\x00\x00\x00\x100\x00?\x00\x00\x00\x021\x00\x06\x00\x00\x00asjdl\x00\x00\x00\x00\x00"
+ENCODED_JSON = '{"x":["y",34,1073741827,2.1223423423356,{"hello":"world","5":[63,"asjdl"]}]}'
+ITERATIONS = 1_00_000
+IMPLEMENTATIONS = ENV.fetch('IMPLEMENTATIONS', 'json,bson,msgpack').split(',').map(&:to_sym)
+
+Viiite.bm do |b|
+  b.variation_point :ruby, Viiite.which_ruby
+  
+  IMPLEMENTATIONS.each do |lib|
+    b.variation_point :lib, lib
+    
+  
+    b.report(:pack) do
+      ITERATIONS.times do
+        case lib
+        when :msgpack then MessagePack.pack(OBJECT_STRUCTURE)
+        when :bson then BSON_IMPL.serialize(OBJECT_STRUCTURE).to_s
+        when :json then OBJECT_STRUCTURE.to_json
+        end
+      end
+    end
+  
+    b.report(:unpack) do
+      ITERATIONS.times do
+        case lib
+        when :msgpack then MessagePack.unpack(ENCODED_MSGPACK)
+        when :bson then BSON_IMPL.deserialize(ENCODED_BSON)
+        when :json then JSON.parse(ENCODED_JSON)
+        end
+      end
+    end
+    
+    b.report(:pack_unpack) do
+      ITERATIONS.times do
+        case lib
+        when :msgpack then MessagePack.unpack(MessagePack.pack(OBJECT_STRUCTURE))
+        when :bson then BSON_IMPL.deserialize(BSON_IMPL.serialize(OBJECT_STRUCTURE).to_s)
+        when :json then JSON.parse(OBJECT_STRUCTURE.to_json)
+        end
+      end
+    end
+
+    b.report(:unpack_pack) do
+      ITERATIONS.times do
+        case lib
+        when :msgpack then MessagePack.pack(MessagePack.unpack(ENCODED_MSGPACK))
+        when :bson then BSON_IMPL.serialize(BSON_IMPL.deserialize(ENCODED_BSON)).to_s
+        when :json then OBJECT_STRUCTURE.to_json(JSON.parse(ENCODED_JSON))
+        end
+      end
+    end
+  end
+end

data/spec/jruby/benchmarks/symbolize_keys_bm.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+$: << File.expand_path('../../../lib', __FILE__)
+
+require 'viiite'
+require 'msgpack'
+
+
+iterations = 10_000
+data = MessagePack.pack(:hello => 'world', :nested => ['structure', {:value => 42}])
+
+Viiite.bm do |b|
+  b.report(:strings) do
+    iterations.times do
+      MessagePack.unpack(data)
+    end
+  end
+
+  b.report(:symbols) do
+    options = {:symbolize_keys => true}
+    iterations.times do
+      MessagePack.unpack(data, options)
+    end
+  end
+end

data/spec/jruby/msgpack/unpacker_spec.rb

@@ -0,0 +1,290 @@
+# encoding: ascii-8bit
+
+require 'stringio'
+require 'tempfile'
+require 'spec_helper'
+
+
+describe ::MessagePack::Unpacker do
+  def flatten(struct, results = [])
+    case struct
+    when Array
+      struct.each { |v| flatten(v, results) }
+    when Hash
+      struct.each { |k, v| flatten(v, flatten(k, results)) }
+    else
+      results << struct
+    end
+    results
+  end
+
+  subject do
+    described_class.new
+  end
+  
+  let :buffer1 do
+    MessagePack.pack(:foo => 'bar')
+  end
+    
+  let :buffer2 do
+    MessagePack.pack(:hello => {:world => [1, 2, 3]})
+  end
+    
+  let :buffer3 do
+    MessagePack.pack(:x => 'y')
+  end
+  
+  describe '#execute/#execute_limit/#finished?' do
+    let :buffer do
+      buffer1 + buffer2 + buffer3
+    end
+    
+    it 'extracts an object from the buffer' do
+      subject.execute(buffer, 0)
+      subject.data.should == {'foo' => 'bar'}
+    end
+
+    it 'extracts an object from the buffer, starting at an offset' do
+      subject.execute(buffer, buffer1.length)
+      subject.data.should == {'hello' => {'world' => [1, 2, 3]}}
+    end
+
+    it 'extracts an object from the buffer, starting at an offset reading bytes up to a limit' do
+      subject.execute_limit(buffer, buffer1.length, buffer2.length)
+      subject.data.should == {'hello' => {'world' => [1, 2, 3]}}
+    end
+    
+    it 'extracts nothing if the limit cuts an object in half' do
+      subject.execute_limit(buffer, buffer1.length, 3)
+      subject.data.should be_nil
+    end
+
+    it 'returns the offset where the object ended' do
+      subject.execute(buffer, 0).should == buffer1.length
+      subject.execute(buffer, buffer1.length).should == buffer1.length + buffer2.length
+    end
+    
+    it 'is finished if #data returns an object' do
+      subject.execute_limit(buffer, buffer1.length, buffer2.length)
+      subject.should be_finished
+      
+      subject.execute_limit(buffer, buffer1.length, 3)
+      subject.should_not be_finished
+    end
+  end
+  
+  describe '#each' do
+    context 'with a buffer' do
+      it 'yields each object in the buffer' do
+        objects = []
+        subject.feed(buffer1)
+        subject.feed(buffer2)
+        subject.feed(buffer3)
+        subject.each do |obj|
+          objects << obj
+        end
+        objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+      end
+      
+      it 'returns an enumerator when no block is given' do
+        subject.feed(buffer1)
+        subject.feed(buffer2)
+        subject.feed(buffer3)
+        enum = subject.each
+        enum.map { |obj| obj.keys.first }.should == %w[foo hello x]
+      end
+    end
+    
+    context 'with a StringIO stream' do
+      it 'yields each object in the stream' do
+        objects = []
+        subject.stream = StringIO.new(buffer1 + buffer2 + buffer3)
+        subject.each do |obj|
+          objects << obj
+        end
+        objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+      end
+    end
+    
+    context 'with a File stream' do
+      it 'yields each object in the stream' do
+        objects = []
+        file = Tempfile.new('msgpack')
+        file.write(buffer1)
+        file.write(buffer2)
+        file.write(buffer3)
+        file.open
+        subject.stream = file
+        subject.each do |obj|
+          objects << obj
+        end
+        objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+      end
+    end
+    
+    context 'with a stream passed to the constructor' do
+      it 'yields each object in the stream' do
+        objects = []
+        unpacker = described_class.new(StringIO.new(buffer1 + buffer2 + buffer3))
+        unpacker.each do |obj|
+          objects << obj
+        end
+        objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+      end
+    end
+  end
+  
+  describe '#feed_each' do
+    it 'feeds the buffer then runs #each' do
+      objects = []
+      subject.feed_each(buffer1 + buffer2 + buffer3) do |obj|
+        objects << obj
+      end
+      objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+    end
+
+    it 'handles chunked data' do
+      objects = []
+      buffer = buffer1 + buffer2 + buffer3
+      buffer.chars.each do |ch|
+        subject.feed_each(ch) do |obj|
+          objects << obj
+        end
+      end
+      objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+    end
+  end
+  
+  describe '#fill' do
+    it 'is a no-op' do
+      subject.stream = StringIO.new(buffer1 + buffer2 + buffer3)
+      subject.fill
+      subject.each { |obj| }
+    end
+  end
+  
+  describe '#reset' do
+    context 'with a buffer' do
+      it 'is unclear what it is supposed to do'
+    end
+    
+    context 'with a stream' do
+      it 'is unclear what it is supposed to do'
+    end
+  end
+
+  context 'regressions' do
+    it 'handles massive arrays (issue #2)' do
+      array = ['foo'] * 10_000
+      MessagePack.unpack(MessagePack.pack(array)).should have(10_000).items
+    end
+  end
+
+  context 'encoding', :encodings do
+    before :all do
+      @default_internal = Encoding.default_internal
+      @default_external = Encoding.default_external
+    end
+
+    after :all do
+      Encoding.default_internal = @default_internal
+      Encoding.default_external = @default_external
+    end
+
+    let :buffer do
+      MessagePack.pack({'hello' => 'world', 'nested' => ['object', {"sk\xC3\xA5l".force_encoding('utf-8') => true}]})
+    end
+
+    let :unpacker do
+      described_class.new
+    end
+
+    before do
+      Encoding.default_internal = Encoding::UTF_8
+      Encoding.default_external = Encoding::ISO_8859_1
+    end
+
+    it 'produces results with default internal encoding' do
+      unpacker.execute(buffer, 0)
+      strings = flatten(unpacker.data).grep(String)
+      strings.map(&:encoding).uniq.should == [Encoding.default_internal]
+    end
+
+    it 'recodes to internal encoding' do
+      unpacker.execute(buffer, 0)
+      unpacker.data['nested'][1].keys.should == ["sk\xC3\xA5l".force_encoding(Encoding.default_internal)]
+    end
+  end
+
+  context 'extensions' do
+    context 'symbolized keys' do
+      let :buffer do
+        MessagePack.pack({'hello' => 'world', 'nested' => ['object', {'structure' => true}]})
+      end
+
+      let :unpacker do
+        described_class.new(:symbolize_keys => true)
+      end
+
+      it 'can symbolize keys when using #execute' do
+        unpacker.execute(buffer, 0)
+        unpacker.data.should == {:hello => 'world', :nested => ['object', {:structure => true}]}
+      end
+
+      it 'can symbolize keys when using #each' do
+        objs = []
+        unpacker.feed(buffer)
+        unpacker.each do |obj|
+          objs << obj
+        end
+        objs.should == [{:hello => 'world', :nested => ['object', {:structure => true}]}]
+      end
+
+      it 'can symbolize keys when using #feed_each' do
+        objs = []
+        unpacker.feed_each(buffer) do |obj|
+          objs << obj
+        end
+        objs.should == [{:hello => 'world', :nested => ['object', {:structure => true}]}]
+      end
+    end
+
+    context 'encoding', :encodings do
+      let :buffer do
+        MessagePack.pack({'hello' => 'world', 'nested' => ['object', {'structure' => true}]})
+      end
+
+      let :unpacker do
+        described_class.new(:encoding => 'UTF-8')
+      end
+
+      it 'can hardcode encoding when using #execute' do
+        unpacker.execute(buffer, 0)
+        strings = flatten(unpacker.data).grep(String)
+        strings.should == %w[hello world nested object structure]
+        strings.map(&:encoding).uniq.should == [Encoding::UTF_8]
+      end
+
+      it 'can hardcode encoding when using #each' do
+        objs = []
+        unpacker.feed(buffer)
+        unpacker.each do |obj|
+          objs << obj
+        end
+        strings = flatten(objs).grep(String)
+        strings.should == %w[hello world nested object structure]
+        strings.map(&:encoding).uniq.should == [Encoding::UTF_8]
+      end
+
+      it 'can hardcode encoding when using #feed_each' do
+        objs = []
+        unpacker.feed_each(buffer) do |obj|
+          objs << obj
+        end
+        strings = flatten(objs).grep(String)
+        strings.should == %w[hello world nested object structure]
+        strings.map(&:encoding).uniq.should == [Encoding::UTF_8]
+      end
+    end
+  end
+end

data/spec/jruby/msgpack_spec.rb

@@ -0,0 +1,136 @@
+# encoding: ascii-8bit
+
+require 'spec_helper'
+
+def utf8enc(str)
+  str.encode('UTF-8')
+end
+
+def asciienc(str)
+  str.encode('ASCII-8BIT')
+end
+
+describe MessagePack do
+  tests = {
+    'constant values' => [
+      ['true', true, "\xC3"],
+      ['false', false, "\xC2"],
+      ['nil', nil, "\xC0"]
+    ],
+    'numbers' => [
+      ['zero', 0, "\x00"],
+      ['127', 127, "\x7F"],
+      ['128', 128, "\xCC\x80"],
+      ['256', 256, "\xCD\x01\x00"],
+      ['-1', -1, "\xFF"],
+      ['-33', -33, "\xD0\xDF"],
+      ['-129', -129, "\xD1\xFF\x7F"],
+      ['small integers', 42, "*"],
+      ['medium integers', 333, "\xCD\x01M"],
+      ['large integers', 2**31 - 1, "\xCE\x7F\xFF\xFF\xFF"],
+      ['huge integers', 2**64 - 1, "\xCF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"],
+      ['negative integers', -1, "\xFF"],
+      ['1.0', 1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"],
+      ['small floats', 3.14, "\xCB@\t\x1E\xB8Q\xEB\x85\x1F"],
+      ['big floats', Math::PI * 1_000_000_000_000_000_000, "\xCBC\xC5\xCC\x96\xEF\xD1\x19%"],
+      ['negative floats', -2.1, "\xCB\xC0\x00\xCC\xCC\xCC\xCC\xCC\xCD"]
+    ],
+    'strings' => [
+      ['strings', utf8enc('hello world'), "\xABhello world"],
+      ['empty strings', utf8enc(''), "\xA0"]
+    ],
+    'arrays' => [
+      ['empty arrays', [], "\x90"],
+      ['arrays with strings', [utf8enc("hello"), utf8enc("world")], "\x92\xA5hello\xA5world"],
+      ['arrays with mixed values', [utf8enc("hello"), utf8enc("world"), 42], "\x93\xA5hello\xA5world*"],
+      ['arrays of arrays', [[[[1, 2], 3], 4]], "\x91\x92\x92\x92\x01\x02\x03\x04"],
+      ['empty arrays', [], "\x90"]
+    ],
+    'hashes' => [
+      ['empty hashes', {}, "\x80"],
+      ['hashes', {utf8enc('foo') => utf8enc('bar')}, "\x81\xA3foo\xA3bar"],
+      ['hashes with mixed keys and values', {utf8enc('foo') => utf8enc('bar'), 3 => utf8enc('three'), utf8enc('four') => 4, utf8enc('x') => [utf8enc('y')], utf8enc('a') => utf8enc('b')}, "\x85\xA3foo\xA3bar\x03\xA5three\xA4four\x04\xA1x\x91\xA1y\xA1a\xA1b"],
+      ['hashes of hashes', {{utf8enc('x') => {utf8enc('y') => utf8enc('z')}} => utf8enc('s')}, "\x81\x81\xA1x\x81\xA1y\xA1z\xA1s"],
+      ['hashes with nils', {utf8enc('foo') => nil}, "\x81\xA3foo\xC0"]
+    ]
+  }
+
+  tests.each do |ctx, its|
+    context("with #{ctx}") do
+      its.each do |desc, unpacked, packed|
+        it("encodes #{desc}") do
+          MessagePack.pack(unpacked).should == packed
+        end
+
+        it "decodes #{desc}" do
+          MessagePack.unpack(packed).should == unpacked
+        end
+      end
+    end
+  end
+
+  context 'using other names for .pack and .unpack' do
+    it 'can unpack with .load' do
+      MessagePack.load("\xABhello world").should == 'hello world'
+    end
+
+    it 'can pack with .dump' do
+      MessagePack.dump(utf8enc('hello world')).should == "\xABhello world"
+    end
+  end
+
+  context 'with symbols' do
+    it 'encodes symbols as strings' do
+      MessagePack.pack(:symbol).should == "\xA6symbol"
+    end
+  end
+
+  context 'with different external encoding', :encodings do
+    before do
+      @default_external = Encoding.default_external
+      @default_internal = Encoding.default_internal
+      Encoding.default_external = Encoding::UTF_8
+      Encoding.default_internal = Encoding::ISO_8859_1
+    end
+
+    after do
+      Encoding.default_external = @default_external
+      Encoding.default_internal = @default_internal
+    end
+
+    it 'transcodes strings when encoding' do
+      input = "sk\xE5l".force_encoding(Encoding::ISO_8859_1)
+      MessagePack.pack(input).should == "\xA5sk\xC3\xA5l"
+    end
+  end
+
+  context 'with other things' do
+    it 'raises an error on #pack with an unsupported type' do
+      expect { MessagePack.pack(self) }.to raise_error(ArgumentError, /^Cannot pack type:/)
+    end
+
+    it 'rasies an error on #unpack with garbage' do
+      skip "???"
+      expect { MessagePack.unpack('asdka;sd') }.to raise_error(MessagePack::UnpackError)
+    end
+  end
+
+  context 'extensions' do
+    it 'can unpack hashes with symbolized keys' do
+      packed = MessagePack.pack({'hello' => 'world', 'nested' => ['object', {'structure' => true}]})
+      unpacked = MessagePack.unpack(packed, :symbolize_keys => true)
+      unpacked.should == {:hello => 'world', :nested => ['object', {:structure => true}]}
+    end
+
+    it 'can unpack strings with a specified encoding', :encodings do
+      packed = MessagePack.pack({utf8enc('hello') => utf8enc('world')})
+      unpacked = MessagePack.unpack(packed)
+      unpacked['hello'].encoding.should == Encoding::UTF_8
+    end
+
+    it 'can pack strings with a specified encoding', :encodings do
+      packed = MessagePack.pack({'hello' => "w\xE5rld".force_encoding(Encoding::ISO_8859_1)})
+      packed.index("w\xC3\xA5rld").should_not be_nil
+    end
+  end
+end