msgpack 0.6.0pre1-x64-mingw32

data/spec/cruby/unpacker_spec.rb

@@ -0,0 +1,305 @@
+# encoding: ascii-8bit
+require 'spec_helper'
+
+describe Unpacker do
+  let :unpacker do
+    Unpacker.new
+  end
+
+  let :packer do
+    Packer.new
+  end
+
+  # TODO initialize
+
+  it 'read_array_header succeeds' do
+    unpacker.feed("\x91")
+    unpacker.read_array_header.should == 1
+  end
+
+  it 'read_array_header fails' do
+    unpacker.feed("\x81")
+    lambda {
+      unpacker.read_array_header
+    }.should raise_error(MessagePack::TypeError)
+  end
+
+  it 'read_map_header converts an map to key-value sequence' do
+    packer.write_array_header(2)
+    packer.write("e")
+    packer.write(1)
+    unpacker = Unpacker.new
+    unpacker.feed(packer.to_s)
+    unpacker.read_array_header.should == 2
+    unpacker.read.should == "e"
+    unpacker.read.should == 1
+  end
+
+  it 'read_map_header succeeds' do
+    unpacker.feed("\x81")
+    unpacker.read_map_header.should == 1
+  end
+
+  it 'read_map_header converts an map to key-value sequence' do
+    packer.write_map_header(1)
+    packer.write("k")
+    packer.write("v")
+    unpacker = Unpacker.new
+    unpacker.feed(packer.to_s)
+    unpacker.read_map_header.should == 1
+    unpacker.read.should == "k"
+    unpacker.read.should == "v"
+  end
+
+  it 'read_map_header fails' do
+    unpacker.feed("\x91")
+    lambda {
+      unpacker.read_map_header
+    }.should raise_error(MessagePack::TypeError)
+  end
+
+  it 'skip_nil succeeds' do
+    unpacker.feed("\xc0")
+    unpacker.skip_nil.should == true
+  end
+
+  it 'skip_nil fails' do
+    unpacker.feed("\x90")
+    unpacker.skip_nil.should == false
+  end
+
+  it 'skip skips objects' do
+    packer.write(1)
+    packer.write(2)
+    packer.write(3)
+    packer.write(4)
+    packer.write(5)
+
+    unpacker = Unpacker.new
+    unpacker.feed(packer.to_s)
+
+    unpacker.read.should == 1
+    unpacker.skip
+    unpacker.read.should == 3
+    unpacker.skip
+    unpacker.read.should == 5
+  end
+
+  it 'read raises EOFError' do
+    lambda {
+      unpacker.read
+    }.should raise_error(EOFError)
+  end
+
+  it 'skip raises EOFError' do
+    lambda {
+      unpacker.skip
+    }.should raise_error(EOFError)
+  end
+
+  it 'skip_nil raises EOFError' do
+    lambda {
+      unpacker.skip_nil
+    }.should raise_error(EOFError)
+  end
+
+  let :sample_object do
+    [1024, {["a","b"]=>["c","d"]}, ["e","f"], "d", 70000, 4.12, 1.5, 1.5, 1.5]
+  end
+
+  it 'feed and each continue internal state' do
+    raw = sample_object.to_msgpack.to_s * 4
+    objects = []
+
+    raw.split(//).each do |b|
+      unpacker.feed(b)
+      unpacker.each {|c|
+        objects << c
+      }
+    end
+
+    objects.should == [sample_object] * 4
+  end
+
+  it 'feed_each continues internal state' do
+    raw = sample_object.to_msgpack.to_s * 4
+    objects = []
+
+    raw.split(//).each do |b|
+      unpacker.feed_each(b) {|c|
+        objects << c
+      }
+    end
+
+    objects.should == [sample_object] * 4
+  end
+
+  it 'reset clears internal buffer' do
+    # 1-element array
+    unpacker.feed("\x91")
+    unpacker.reset
+    unpacker.feed("\x01")
+
+    unpacker.each.map {|x| x }.should == [1]
+  end
+
+  it 'reset clears internal state' do
+    # 1-element array
+    unpacker.feed("\x91")
+    unpacker.each.map {|x| x }.should == []
+
+    unpacker.reset
+
+    unpacker.feed("\x01")
+    unpacker.each.map {|x| x }.should == [1]
+  end
+
+  it 'frozen short strings' do
+    raw = sample_object.to_msgpack.to_s.force_encoding('UTF-8')
+    lambda {
+      unpacker.feed_each(raw.freeze) { }
+    }.should_not raise_error
+  end
+
+  it 'frozen long strings' do
+    raw = (sample_object.to_msgpack.to_s * 10240).force_encoding('UTF-8')
+    lambda {
+      unpacker.feed_each(raw.freeze) { }
+    }.should_not raise_error
+  end
+
+  it 'read raises level stack too deep error' do
+    512.times { packer.write_array_header(1) }
+    packer.write(nil)
+
+    unpacker = Unpacker.new
+    unpacker.feed(packer.to_s)
+    lambda {
+      unpacker.read
+    }.should raise_error(MessagePack::StackError)
+  end
+
+  it 'skip raises level stack too deep error' do
+    512.times { packer.write_array_header(1) }
+    packer.write(nil)
+
+    unpacker = Unpacker.new
+    unpacker.feed(packer.to_s)
+    lambda {
+      unpacker.skip
+    }.should raise_error(MessagePack::StackError)
+  end
+
+  it 'read raises invalid byte error' do
+    unpacker.feed("\xc1")
+    lambda {
+      unpacker.read
+    }.should raise_error(MessagePack::MalformedFormatError)
+  end
+
+  it 'skip raises invalid byte error' do
+    unpacker.feed("\xc1")
+    lambda {
+      unpacker.skip
+    }.should raise_error(MessagePack::MalformedFormatError)
+  end
+
+  it "gc mark" do
+    raw = sample_object.to_msgpack.to_s * 4
+
+    n = 0
+    raw.split(//).each do |b|
+      GC.start
+      unpacker.feed_each(b) {|o|
+        GC.start
+        o.should == sample_object
+        n += 1
+      }
+      GC.start
+    end
+
+    n.should == 4
+  end
+
+  it "buffer" do
+    orig = "a"*32*1024*4
+    raw = orig.to_msgpack.to_s
+
+    n = 655
+    times = raw.size / n
+    times += 1 unless raw.size % n == 0
+
+    off = 0
+    parsed = false
+
+    times.times do
+      parsed.should == false
+
+      seg = raw[off, n]
+      off += seg.length
+
+      unpacker.feed_each(seg) {|obj|
+        parsed.should == false
+        obj.should == orig
+        parsed = true
+      }
+    end
+
+    parsed.should == true
+  end
+
+  it 'MessagePack.unpack symbolize_keys' do
+    symbolized_hash = {:a => 'b', :c => 'd'}
+    MessagePack.load(MessagePack.pack(symbolized_hash), :symbolize_keys => true).should == symbolized_hash
+    MessagePack.unpack(MessagePack.pack(symbolized_hash), :symbolize_keys => true).should == symbolized_hash
+  end
+
+  it 'Unpacker#unpack symbolize_keys' do
+    unpacker = Unpacker.new(:symbolize_keys => true)
+    symbolized_hash = {:a => 'b', :c => 'd'}
+    unpacker.feed(MessagePack.pack(symbolized_hash)).read.should == symbolized_hash
+  end
+
+  it "msgpack str 8 type" do
+    MessagePack.unpack([0xd9, 0x00].pack('C*')).should == ""
+    MessagePack.unpack([0xd9, 0x00].pack('C*')).encoding.should == Encoding::UTF_8
+    MessagePack.unpack([0xd9, 0x01].pack('C*') + 'a').should == "a"
+    MessagePack.unpack([0xd9, 0x02].pack('C*') + 'aa').should == "aa"
+  end
+
+  it "msgpack str 16 type" do
+    MessagePack.unpack([0xda, 0x00, 0x00].pack('C*')).should == ""
+    MessagePack.unpack([0xda, 0x00, 0x00].pack('C*')).encoding.should == Encoding::UTF_8
+    MessagePack.unpack([0xda, 0x00, 0x01].pack('C*') + 'a').should == "a"
+    MessagePack.unpack([0xda, 0x00, 0x02].pack('C*') + 'aa').should == "aa"
+  end
+
+  it "msgpack str 32 type" do
+    MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x00].pack('C*')).should == ""
+    MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x00].pack('C*')).encoding.should == Encoding::UTF_8
+    MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x01].pack('C*') + 'a').should == "a"
+    MessagePack.unpack([0xdb, 0x00, 0x00, 0x00, 0x02].pack('C*') + 'aa').should == "aa"
+  end
+
+  it "msgpack bin 8 type" do
+    MessagePack.unpack([0xc4, 0x00].pack('C*')).should == ""
+    MessagePack.unpack([0xc4, 0x00].pack('C*')).encoding.should == Encoding::ASCII_8BIT
+    MessagePack.unpack([0xc4, 0x01].pack('C*') + 'a').should == "a"
+    MessagePack.unpack([0xc4, 0x02].pack('C*') + 'aa').should == "aa"
+  end
+
+  it "msgpack bin 16 type" do
+    MessagePack.unpack([0xc5, 0x00, 0x00].pack('C*')).should == ""
+    MessagePack.unpack([0xc5, 0x00, 0x00].pack('C*')).encoding.should == Encoding::ASCII_8BIT
+    MessagePack.unpack([0xc5, 0x00, 0x01].pack('C*') + 'a').should == "a"
+    MessagePack.unpack([0xc5, 0x00, 0x02].pack('C*') + 'aa').should == "aa"
+  end
+
+  it "msgpack bin 32 type" do
+    MessagePack.unpack([0xc6, 0x00, 0x00, 0x00, 0x00].pack('C*')).should == ""
+    MessagePack.unpack([0xc6, 0x0, 0x00, 0x00, 0x000].pack('C*')).encoding.should == Encoding::ASCII_8BIT
+    MessagePack.unpack([0xc6, 0x00, 0x00, 0x00, 0x01].pack('C*') + 'a').should == "a"
+    MessagePack.unpack([0xc6, 0x00, 0x00, 0x00, 0x02].pack('C*') + 'aa').should == "aa"
+  end
+end
+

data/spec/format_spec.rb

@@ -0,0 +1,282 @@
+# encoding: ascii-8bit
+require 'spec_helper'
+
+describe MessagePack do
+  it "nil" do
+    check 1, nil
+  end
+
+  it "true" do
+    check 1, true
+  end
+
+  it "false" do
+    check 1, false
+  end
+
+  it "zero" do
+    check 1, 0
+  end
+
+  it "positive fixnum" do
+    check 1, 1
+    check 1, (1 << 6)
+    check 1, (1 << 7)-1
+  end
+
+  it "positive int 8" do
+    check 1, -1
+    check 2, (1 << 7)
+    check 2, (1 << 8) - 1
+  end
+
+  it "positive int 16" do
+    check 3, (1 << 8)
+    check 3, (1 << 16) - 1
+  end
+
+  it "positive int 32" do
+    check 5, (1 << 16)
+    check 5, (1 << 32) - 1
+  end
+
+  it "positive int 64" do
+    check 9, (1 << 32)
+    #check 9, (1<<64)-1
+  end
+
+  it "negative fixnum" do
+    check 1, -1
+    check 1, -((1 << 5)-1)
+    check 1, -(1 << 5)
+  end
+
+  it "negative int 8" do
+    check 2, -((1 << 5)+1)
+    check 2, -(1 << 7)
+  end
+
+  it "negative int 16" do
+    check 3, -((1 << 7)+1)
+    check 3, -(1 << 15)
+  end
+
+  it "negative int 32" do
+    check 5, -((1 << 15)+1)
+    check 5, -(1 << 31)
+  end
+
+  it "negative int 64" do
+    check 9, -((1 << 31)+1)
+    check 9, -(1 << 63)
+  end
+
+  it "double" do
+    check 9, 1.0
+    check 9, 0.1
+    check 9, -0.1
+    check 9, -1.0
+  end
+
+  it "fixraw" do
+    check_raw 1, 0
+    check_raw 1, (1 << 5)-1
+  end
+
+  it "raw 8" do
+    check_raw 2, (1 << 5)
+    check_raw 2, (1 << 8)-1
+  end
+
+  it "raw 16" do
+    check_raw 3, (1 << 8)
+    check_raw 3, (1 << 16)-1
+  end
+
+  it "raw 32" do
+    check_raw 5, (1 << 16)
+    #check_raw 5, (1 << 32)-1  # memory error
+  end
+
+  it "str encoding is UTF_8" do
+    v = pack_unpack('string'.force_encoding(Encoding::UTF_8))
+    v.encoding.should == Encoding::UTF_8
+  end
+
+  it "str transcode US-ASCII" do
+    v = pack_unpack('string'.force_encoding(Encoding::US_ASCII))
+    v.encoding.should == Encoding::UTF_8
+  end
+
+  it "str transcode UTF-16" do
+    v = pack_unpack('string'.encode(Encoding::UTF_16))
+    v.encoding.should == Encoding::UTF_8
+    v.should == 'string'
+  end
+
+  it "str transcode EUC-JP 7bit safe" do
+    v = pack_unpack('string'.force_encoding(Encoding::EUC_JP))
+    v.encoding.should == Encoding::UTF_8
+    v.should == 'string'
+  end
+
+  it "str transcode EUC-JP 7bit unsafe" do
+    v = pack_unpack([0xa4, 0xa2].pack('C*').force_encoding(Encoding::EUC_JP))
+    v.encoding.should == Encoding::UTF_8
+    v.should == "\xE3\x81\x82".force_encoding('UTF-8')
+  end
+
+  it "bin 8" do
+    check_bin 2, (1<<8)-1
+  end
+
+  it "bin 16" do
+    check_bin 3, (1<<16)-1
+  end
+
+  it "bin 32" do
+    check_bin 5, (1<<16)
+  end
+
+  it "bin encoding is ASCII_8BIT" do
+    pack_unpack('string'.force_encoding(Encoding::ASCII_8BIT)).encoding.should == Encoding::ASCII_8BIT
+  end
+
+  it "fixarray" do
+    check_array 1, 0
+    check_array 1, (1 << 4)-1
+  end
+
+  it "array 16" do
+    check_array 3, (1 << 4)
+    #check_array 3, (1 << 16)-1
+  end
+
+  it "array 32" do
+    #check_array 5, (1 << 16)
+    #check_array 5, (1 << 32)-1  # memory error
+  end
+
+  it "nil" do
+    match nil, "\xc0"
+  end
+
+  it "false" do
+    match false, "\xc2"
+  end
+
+  it "true" do
+    match true, "\xc3"
+  end
+
+  it "0" do
+    match 0, "\x00"
+  end
+
+  it "127" do
+    match 127, "\x7f"
+  end
+
+  it "128" do
+    match 128, "\xcc\x80"
+  end
+
+  it "256" do
+    match 256, "\xcd\x01\x00"
+  end
+
+  it "-1" do
+    match -1, "\xff"
+  end
+
+  it "-33" do
+    match -33, "\xd0\xdf"
+  end
+
+  it "-129" do
+    match -129, "\xd1\xff\x7f"
+  end
+
+  it "{1=>1}" do
+    obj = {1=>1}
+    match obj, "\x81\x01\x01"
+  end
+
+  it "1.0" do
+    match 1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"
+  end
+
+  it "[]" do
+    match [], "\x90"
+  end
+
+  it "[0, 1, ..., 14]" do
+    obj = (0..14).to_a
+    match obj, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"
+  end
+
+  it "[0, 1, ..., 15]" do
+    obj = (0..15).to_a
+    match obj, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+  end
+
+  it "{}" do
+    obj = {}
+    match obj, "\x80"
+  end
+
+## FIXME
+#  it "{0=>0, 1=>1, ..., 14=>14}" do
+#    a = (0..14).to_a;
+#    match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"
+#  end
+#
+#  it "{0=>0, 1=>1, ..., 15=>15}" do
+#    a = (0..15).to_a;
+#    match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"
+#  end
+
+## FIXME
+#  it "fixmap" do
+#    check_map 1, 0
+#    check_map 1, (1<<4)-1
+#  end
+#
+#  it "map 16" do
+#    check_map 3, (1<<4)
+#    check_map 3, (1<<16)-1
+#  end
+#
+#  it "map 32" do
+#    check_map 5, (1<<16)
+#    #check_map 5, (1<<32)-1  # memory error
+#  end
+
+  def check(len, obj)
+    raw = obj.to_msgpack.to_s
+    raw.length.should == len
+    MessagePack.unpack(raw).should == obj
+  end
+
+  def check_raw(overhead, num)
+    check num+overhead, (" "*num).force_encoding(Encoding::UTF_8)
+  end
+
+  def check_bin(overhead, num)
+    check num+overhead, (" "*num).force_encoding(Encoding::ASCII_8BIT)
+  end
+
+  def check_array(overhead, num)
+    check num+overhead, Array.new(num)
+  end
+
+  def match(obj, buf)
+    raw = obj.to_msgpack.to_s
+    raw.should == buf
+  end
+
+  def pack_unpack(obj)
+    MessagePack.unpack(obj.to_msgpack)
+  end
+end
+

data/spec/jruby/benchmarks/shootout_bm.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+
+if RUBY_PLATFORM.include?('java')
+  # JRuby should use this library, MRI should use the standard gem
+  $: << File.expand_path('../../../lib', __FILE__)
+end
+
+require 'viiite'
+require 'msgpack'
+require 'json'
+require 'bson'
+
+if RUBY_PLATFORM.include?('java')
+  BSON_IMPL = BSON::BSON_JAVA
+else
+  BSON_IMPL = BSON::BSON_C
+end
+
+OBJECT_STRUCTURE = {'x' => ['y', 34, 2**30 + 3, 2.1223423423356, {'hello' => 'world', '5' => [63, 'asjdl']}]}
+ENCODED_MSGPACK = "\x81\xA1x\x95\xA1y\"\xCE@\x00\x00\x03\xCB@\x00\xFA\x8E\x9F9\xFA\xC1\x82\xA5hello\xA5world\xA15\x92?\xA5asjdl"
+ENCODED_BSON = "d\x00\x00\x00\x04x\x00\\\x00\x00\x00\x020\x00\x02\x00\x00\x00y\x00\x101\x00\"\x00\x00\x00\x102\x00\x03\x00\x00@\x013\x00\xC1\xFA9\x9F\x8E\xFA\x00@\x034\x002\x00\x00\x00\x02hello\x00\x06\x00\x00\x00world\x00\x045\x00\x19\x00\x00\x00\x100\x00?\x00\x00\x00\x021\x00\x06\x00\x00\x00asjdl\x00\x00\x00\x00\x00"
+ENCODED_JSON = '{"x":["y",34,1073741827,2.1223423423356,{"hello":"world","5":[63,"asjdl"]}]}'
+ITERATIONS = 1_00_000
+IMPLEMENTATIONS = ENV.fetch('IMPLEMENTATIONS', 'json,bson,msgpack').split(',').map(&:to_sym)
+
+Viiite.bm do |b|
+  b.variation_point :ruby, Viiite.which_ruby
+  
+  IMPLEMENTATIONS.each do |lib|
+    b.variation_point :lib, lib
+    
+  
+    b.report(:pack) do
+      ITERATIONS.times do
+        case lib
+        when :msgpack then MessagePack.pack(OBJECT_STRUCTURE)
+        when :bson then BSON_IMPL.serialize(OBJECT_STRUCTURE).to_s
+        when :json then OBJECT_STRUCTURE.to_json
+        end
+      end
+    end
+  
+    b.report(:unpack) do
+      ITERATIONS.times do
+        case lib
+        when :msgpack then MessagePack.unpack(ENCODED_MSGPACK)
+        when :bson then BSON_IMPL.deserialize(ENCODED_BSON)
+        when :json then JSON.parse(ENCODED_JSON)
+        end
+      end
+    end
+    
+    b.report(:pack_unpack) do
+      ITERATIONS.times do
+        case lib
+        when :msgpack then MessagePack.unpack(MessagePack.pack(OBJECT_STRUCTURE))
+        when :bson then BSON_IMPL.deserialize(BSON_IMPL.serialize(OBJECT_STRUCTURE).to_s)
+        when :json then JSON.parse(OBJECT_STRUCTURE.to_json)
+        end
+      end
+    end
+
+    b.report(:unpack_pack) do
+      ITERATIONS.times do
+        case lib
+        when :msgpack then MessagePack.pack(MessagePack.unpack(ENCODED_MSGPACK))
+        when :bson then BSON_IMPL.serialize(BSON_IMPL.deserialize(ENCODED_BSON)).to_s
+        when :json then OBJECT_STRUCTURE.to_json(JSON.parse(ENCODED_JSON))
+        end
+      end
+    end
+  end
+end

data/spec/jruby/benchmarks/symbolize_keys_bm.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+$: << File.expand_path('../../../lib', __FILE__)
+
+require 'viiite'
+require 'msgpack'
+
+
+iterations = 10_000
+data = MessagePack.pack(:hello => 'world', :nested => ['structure', {:value => 42}])
+
+Viiite.bm do |b|
+  b.report(:strings) do
+    iterations.times do
+      MessagePack.unpack(data)
+    end
+  end
+
+  b.report(:symbols) do
+    options = {:symbolize_keys => true}
+    iterations.times do
+      MessagePack.unpack(data, options)
+    end
+  end
+end