msgpack 1.3.3 → 1.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yaml +57 -0
- data/.rubocop.yml +2 -2
- data/ChangeLog +74 -0
- data/Gemfile +1 -1
- data/README.md +266 -0
- data/Rakefile +1 -9
- data/bench/bench.rb +78 -0
- data/bin/console +8 -0
- data/doclib/msgpack/factory.rb +47 -3
- data/doclib/msgpack/packer.rb +5 -4
- data/doclib/msgpack/unpacker.rb +2 -2
- data/ext/java/org/msgpack/jruby/Buffer.java +23 -16
- data/ext/java/org/msgpack/jruby/Decoder.java +46 -23
- data/ext/java/org/msgpack/jruby/Encoder.java +68 -30
- data/ext/java/org/msgpack/jruby/ExtensionRegistry.java +37 -49
- data/ext/java/org/msgpack/jruby/ExtensionValue.java +5 -8
- data/ext/java/org/msgpack/jruby/Factory.java +47 -7
- data/ext/java/org/msgpack/jruby/Packer.java +29 -17
- data/ext/java/org/msgpack/jruby/Unpacker.java +72 -37
- data/ext/msgpack/buffer.c +42 -68
- data/ext/msgpack/buffer.h +59 -14
- data/ext/msgpack/buffer_class.c +90 -52
- data/ext/msgpack/compat.h +1 -111
- data/ext/msgpack/extconf.rb +45 -19
- data/ext/msgpack/factory_class.c +133 -43
- data/ext/msgpack/packer.c +60 -36
- data/ext/msgpack/packer.h +27 -25
- data/ext/msgpack/packer_class.c +84 -77
- data/ext/msgpack/packer_class.h +11 -0
- data/ext/msgpack/packer_ext_registry.c +24 -32
- data/ext/msgpack/packer_ext_registry.h +40 -33
- data/ext/msgpack/sysdep.h +5 -2
- data/ext/msgpack/unpacker.c +132 -115
- data/ext/msgpack/unpacker.h +23 -10
- data/ext/msgpack/unpacker_class.c +83 -78
- data/ext/msgpack/unpacker_class.h +11 -0
- data/ext/msgpack/unpacker_ext_registry.c +42 -18
- data/ext/msgpack/unpacker_ext_registry.h +23 -16
- data/lib/msgpack/bigint.rb +69 -0
- data/lib/msgpack/factory.rb +103 -0
- data/lib/msgpack/symbol.rb +21 -4
- data/lib/msgpack/time.rb +1 -1
- data/lib/msgpack/version.rb +4 -8
- data/lib/msgpack.rb +6 -12
- data/msgpack.gemspec +4 -6
- data/spec/bigint_spec.rb +26 -0
- data/spec/cruby/buffer_spec.rb +17 -0
- data/spec/factory_spec.rb +351 -12
- data/spec/msgpack_spec.rb +1 -1
- data/spec/packer_spec.rb +18 -0
- data/spec/spec_helper.rb +37 -3
- data/spec/timestamp_spec.rb +38 -0
- data/spec/unpacker_spec.rb +157 -4
- metadata +31 -61
- data/.travis.yml +0 -43
- data/README.rdoc +0 -225
- data/bench/pack.rb +0 -23
- data/bench/pack_log.rb +0 -33
- data/bench/pack_log_long.rb +0 -65
- data/bench/pack_symbols.rb +0 -28
- data/bench/run.sh +0 -14
- data/bench/run_long.sh +0 -35
- data/bench/run_symbols.sh +0 -26
- data/bench/unpack.rb +0 -21
- data/bench/unpack_log.rb +0 -34
- data/bench/unpack_log_long.rb +0 -67
data/spec/unpacker_spec.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: ascii-8bit
|
2
|
-
|
3
1
|
require 'stringio'
|
4
2
|
require 'tempfile'
|
5
3
|
require 'zlib'
|
@@ -18,13 +16,35 @@ describe MessagePack::Unpacker do
|
|
18
16
|
it 'gets options to specify how to unpack values' do
|
19
17
|
u1 = MessagePack::Unpacker.new
|
20
18
|
u1.symbolize_keys?.should == false
|
19
|
+
u1.freeze?.should == false
|
21
20
|
u1.allow_unknown_ext?.should == false
|
22
21
|
|
23
|
-
u2 = MessagePack::Unpacker.new(symbolize_keys: true, allow_unknown_ext: true)
|
22
|
+
u2 = MessagePack::Unpacker.new(symbolize_keys: true, freeze: true, allow_unknown_ext: true)
|
24
23
|
u2.symbolize_keys?.should == true
|
24
|
+
u2.freeze?.should == true
|
25
25
|
u2.allow_unknown_ext?.should == true
|
26
26
|
end
|
27
27
|
|
28
|
+
if automatic_string_keys_deduplication?
|
29
|
+
it 'ensure string hash keys are deduplicated' do
|
30
|
+
sample_data = [{"foo" => 1}, {"foo" => 2}]
|
31
|
+
sample_packed = MessagePack.pack(sample_data).force_encoding('ASCII-8BIT')
|
32
|
+
unpacker.feed(sample_packed)
|
33
|
+
hashes = nil
|
34
|
+
unpacker.each { |obj| hashes = obj }
|
35
|
+
expect(hashes[0].keys.first).to equal(hashes[1].keys.first)
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'ensure strings are not deduplicated' do
|
39
|
+
sample_data = ["foo"]
|
40
|
+
sample_packed = MessagePack.pack(sample_data).force_encoding('ASCII-8BIT')
|
41
|
+
unpacker.feed(sample_packed)
|
42
|
+
ary = nil
|
43
|
+
unpacker.each { |obj| ary = obj }
|
44
|
+
expect(ary.first.frozen?).to eq(false)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
28
48
|
it 'gets IO or object which has #read to read data from it' do
|
29
49
|
sample_data = {"message" => "morning!", "num" => 1}
|
30
50
|
sample_packed = MessagePack.pack(sample_data).force_encoding('ASCII-8BIT')
|
@@ -280,6 +300,21 @@ describe MessagePack::Unpacker do
|
|
280
300
|
MessagePack.unpack(MessagePack.pack(symbolized_hash), :symbolize_keys => true).should == symbolized_hash
|
281
301
|
end
|
282
302
|
|
303
|
+
it 'MessagePack.unpack symbolize_keys preserve encoding' do
|
304
|
+
hash = { :ascii => 1, :utf8_é => 2}
|
305
|
+
loaded_hash = MessagePack.load(MessagePack.pack(hash), :symbolize_keys => true)
|
306
|
+
|
307
|
+
hash.keys[0].encoding.should == Encoding::US_ASCII # Ruby coerce symbols to US-ASCII when possible.
|
308
|
+
loaded_hash.keys[0].should == hash.keys[0]
|
309
|
+
loaded_hash.keys[0].encoding.should == hash.keys[0].encoding
|
310
|
+
|
311
|
+
hash.keys[1].encoding.should == Encoding::UTF_8
|
312
|
+
loaded_hash.keys[1].should == hash.keys[1]
|
313
|
+
loaded_hash.keys[1].encoding.should == hash.keys[1].encoding
|
314
|
+
|
315
|
+
MessagePack.unpack(MessagePack.pack(hash), :symbolize_keys => true).should == hash
|
316
|
+
end
|
317
|
+
|
283
318
|
it 'Unpacker#unpack symbolize_keys' do
|
284
319
|
unpacker = MessagePack::Unpacker.new(:symbolize_keys => true)
|
285
320
|
symbolized_hash = {:a => 'b', :c => 'd'}
|
@@ -621,6 +656,24 @@ describe MessagePack::Unpacker do
|
|
621
656
|
array = ['foo'] * 10_000
|
622
657
|
MessagePack.unpack(MessagePack.pack(array)).size.should == 10_000
|
623
658
|
end
|
659
|
+
|
660
|
+
it 'preserves string encoding (issue #200)' do
|
661
|
+
string = 'a'.force_encoding(Encoding::UTF_8)
|
662
|
+
MessagePack.unpack(MessagePack.pack(string)).encoding.should == string.encoding
|
663
|
+
|
664
|
+
string *= 256
|
665
|
+
MessagePack.unpack(MessagePack.pack(string)).encoding.should == string.encoding
|
666
|
+
end
|
667
|
+
|
668
|
+
it 'returns correct size for array16 (issue #127)' do
|
669
|
+
unpacker.feed("\xdc\x00\x01\x01")
|
670
|
+
unpacker.read_array_header.should == 1
|
671
|
+
end
|
672
|
+
|
673
|
+
it 'returns correct size for map16 (issue #127)' do
|
674
|
+
unpacker.feed("\xde\x00\x02\x01\x02\x03\x04")
|
675
|
+
unpacker.read_map_header.should == 2
|
676
|
+
end
|
624
677
|
end
|
625
678
|
|
626
679
|
context 'extensions' do
|
@@ -651,9 +704,109 @@ describe MessagePack::Unpacker do
|
|
651
704
|
end
|
652
705
|
end
|
653
706
|
|
707
|
+
context 'freeze' do
|
708
|
+
let :struct do
|
709
|
+
{'hello' => 'world', 'nested' => ['object', {'structure' => true}]}
|
710
|
+
end
|
711
|
+
|
712
|
+
let :buffer do
|
713
|
+
MessagePack.pack(struct)
|
714
|
+
end
|
715
|
+
|
716
|
+
let :unpacker do
|
717
|
+
described_class.new(:freeze => true)
|
718
|
+
end
|
719
|
+
|
720
|
+
if (-"test").equal?(-"test") # RUBY_VERSION >= "2.5"
|
721
|
+
it 'dedups strings' do
|
722
|
+
interned_str = -"test"
|
723
|
+
roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
|
724
|
+
expect(roundtrip).to be interned_str
|
725
|
+
|
726
|
+
interned_str = -""
|
727
|
+
roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
|
728
|
+
expect(roundtrip).to be interned_str
|
729
|
+
end
|
730
|
+
end
|
731
|
+
|
732
|
+
it 'can freeze objects when using .unpack' do
|
733
|
+
parsed_struct = MessagePack.unpack(buffer, freeze: true)
|
734
|
+
parsed_struct.should == struct
|
735
|
+
|
736
|
+
parsed_struct.should be_frozen
|
737
|
+
parsed_struct['hello'].should be_frozen
|
738
|
+
parsed_struct['nested'].should be_frozen
|
739
|
+
parsed_struct['nested'][0].should be_frozen
|
740
|
+
parsed_struct['nested'][1].should be_frozen
|
741
|
+
|
742
|
+
if string_deduplication?
|
743
|
+
parsed_struct.keys[0].should be_equal('hello'.freeze)
|
744
|
+
parsed_struct.keys[1].should be_equal('nested'.freeze)
|
745
|
+
parsed_struct.values[0].should be_equal('world'.freeze)
|
746
|
+
parsed_struct.values[1][0].should be_equal('object'.freeze)
|
747
|
+
parsed_struct.values[1][1].keys[0].should be_equal('structure'.freeze)
|
748
|
+
end
|
749
|
+
end
|
750
|
+
|
751
|
+
it 'can freeze objects when using #each' do
|
752
|
+
objs = []
|
753
|
+
unpacker.feed(buffer)
|
754
|
+
unpacker.each do |obj|
|
755
|
+
objs << obj
|
756
|
+
end
|
757
|
+
|
758
|
+
parsed_struct = objs.first
|
759
|
+
parsed_struct.should == struct
|
760
|
+
|
761
|
+
parsed_struct.should be_frozen
|
762
|
+
parsed_struct['hello'].should be_frozen
|
763
|
+
parsed_struct['nested'].should be_frozen
|
764
|
+
parsed_struct['nested'][0].should be_frozen
|
765
|
+
parsed_struct['nested'][1].should be_frozen
|
766
|
+
|
767
|
+
if string_deduplication?
|
768
|
+
parsed_struct.keys[0].should be_equal('hello'.freeze)
|
769
|
+
parsed_struct.keys[1].should be_equal('nested'.freeze)
|
770
|
+
parsed_struct.values[0].should be_equal('world'.freeze)
|
771
|
+
parsed_struct.values[1][0].should be_equal('object'.freeze)
|
772
|
+
parsed_struct.values[1][1].keys[0].should be_equal('structure'.freeze)
|
773
|
+
end
|
774
|
+
end
|
775
|
+
|
776
|
+
it 'can freeze objects when using #feed_each' do
|
777
|
+
objs = []
|
778
|
+
unpacker.feed_each(buffer) do |obj|
|
779
|
+
objs << obj
|
780
|
+
end
|
781
|
+
|
782
|
+
parsed_struct = objs.first
|
783
|
+
parsed_struct.should == struct
|
784
|
+
|
785
|
+
parsed_struct.should be_frozen
|
786
|
+
parsed_struct['hello'].should be_frozen
|
787
|
+
parsed_struct['nested'].should be_frozen
|
788
|
+
parsed_struct['nested'][0].should be_frozen
|
789
|
+
parsed_struct['nested'][1].should be_frozen
|
790
|
+
|
791
|
+
if string_deduplication?
|
792
|
+
parsed_struct.keys[0].should be_equal('hello'.freeze)
|
793
|
+
parsed_struct.keys[1].should be_equal('nested'.freeze)
|
794
|
+
parsed_struct.values[0].should be_equal('world'.freeze)
|
795
|
+
parsed_struct.values[1][0].should be_equal('object'.freeze)
|
796
|
+
parsed_struct.values[1][1].keys[0].should be_equal('structure'.freeze)
|
797
|
+
end
|
798
|
+
end
|
799
|
+
end
|
800
|
+
|
654
801
|
context 'binary encoding', :encodings do
|
655
802
|
let :buffer do
|
656
|
-
MessagePack.pack({
|
803
|
+
MessagePack.pack({
|
804
|
+
'hello'.b => 'world'.b,
|
805
|
+
'nested'.b => [
|
806
|
+
'object'.b,
|
807
|
+
{'structure'.b => true},
|
808
|
+
]
|
809
|
+
})
|
657
810
|
end
|
658
811
|
|
659
812
|
let :unpacker do
|
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: msgpack
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
- Theo Hultberg
|
9
9
|
- Satoshi Tagomori
|
10
|
-
autorequire:
|
10
|
+
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2022-09-30 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|
@@ -44,30 +44,16 @@ dependencies:
|
|
44
44
|
name: rake-compiler
|
45
45
|
requirement: !ruby/object:Gem::Requirement
|
46
46
|
requirements:
|
47
|
-
- - "
|
48
|
-
- !ruby/object:Gem::Version
|
49
|
-
version: '1.0'
|
50
|
-
type: :development
|
51
|
-
prerelease: false
|
52
|
-
version_requirements: !ruby/object:Gem::Requirement
|
53
|
-
requirements:
|
54
|
-
- - "~>"
|
55
|
-
- !ruby/object:Gem::Version
|
56
|
-
version: '1.0'
|
57
|
-
- !ruby/object:Gem::Dependency
|
58
|
-
name: rake-compiler-dock
|
59
|
-
requirement: !ruby/object:Gem::Requirement
|
60
|
-
requirements:
|
61
|
-
- - "~>"
|
47
|
+
- - ">="
|
62
48
|
- !ruby/object:Gem::Version
|
63
|
-
version:
|
49
|
+
version: 1.1.9
|
64
50
|
type: :development
|
65
51
|
prerelease: false
|
66
52
|
version_requirements: !ruby/object:Gem::Requirement
|
67
53
|
requirements:
|
68
|
-
- - "
|
54
|
+
- - ">="
|
69
55
|
- !ruby/object:Gem::Version
|
70
|
-
version:
|
56
|
+
version: 1.1.9
|
71
57
|
- !ruby/object:Gem::Dependency
|
72
58
|
name: rspec
|
73
59
|
requirement: !ruby/object:Gem::Requirement
|
@@ -110,6 +96,20 @@ dependencies:
|
|
110
96
|
- - ">="
|
111
97
|
- !ruby/object:Gem::Version
|
112
98
|
version: '0'
|
99
|
+
- !ruby/object:Gem::Dependency
|
100
|
+
name: benchmark-ips
|
101
|
+
requirement: !ruby/object:Gem::Requirement
|
102
|
+
requirements:
|
103
|
+
- - "~>"
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
version: 2.10.0
|
106
|
+
type: :development
|
107
|
+
prerelease: false
|
108
|
+
version_requirements: !ruby/object:Gem::Requirement
|
109
|
+
requirements:
|
110
|
+
- - "~>"
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: 2.10.0
|
113
113
|
description: MessagePack is a binary-based efficient object serialization library.
|
114
114
|
It enables to exchange structured objects between many languages like JSON. But
|
115
115
|
unlike JSON, it is very fast and small.
|
@@ -122,25 +122,17 @@ extensions:
|
|
122
122
|
- ext/msgpack/extconf.rb
|
123
123
|
extra_rdoc_files: []
|
124
124
|
files:
|
125
|
+
- ".github/workflows/ci.yaml"
|
125
126
|
- ".gitignore"
|
126
127
|
- ".rubocop.yml"
|
127
|
-
- ".travis.yml"
|
128
128
|
- ChangeLog
|
129
129
|
- Gemfile
|
130
130
|
- LICENSE
|
131
|
-
- README.
|
131
|
+
- README.md
|
132
132
|
- Rakefile
|
133
133
|
- appveyor.yml
|
134
|
-
- bench/
|
135
|
-
-
|
136
|
-
- bench/pack_log_long.rb
|
137
|
-
- bench/pack_symbols.rb
|
138
|
-
- bench/run.sh
|
139
|
-
- bench/run_long.sh
|
140
|
-
- bench/run_symbols.sh
|
141
|
-
- bench/unpack.rb
|
142
|
-
- bench/unpack_log.rb
|
143
|
-
- bench/unpack_log_long.rb
|
134
|
+
- bench/bench.rb
|
135
|
+
- bin/console
|
144
136
|
- doclib/msgpack.rb
|
145
137
|
- doclib/msgpack/buffer.rb
|
146
138
|
- doclib/msgpack/core_ext.rb
|
@@ -190,6 +182,7 @@ files:
|
|
190
182
|
- ext/msgpack/unpacker_ext_registry.c
|
191
183
|
- ext/msgpack/unpacker_ext_registry.h
|
192
184
|
- lib/msgpack.rb
|
185
|
+
- lib/msgpack/bigint.rb
|
193
186
|
- lib/msgpack/core_ext.rb
|
194
187
|
- lib/msgpack/factory.rb
|
195
188
|
- lib/msgpack/packer.rb
|
@@ -200,6 +193,7 @@ files:
|
|
200
193
|
- lib/msgpack/version.rb
|
201
194
|
- msgpack.gemspec
|
202
195
|
- msgpack.org.md
|
196
|
+
- spec/bigint_spec.rb
|
203
197
|
- spec/cases.json
|
204
198
|
- spec/cases.msg
|
205
199
|
- spec/cases_compact.msg
|
@@ -228,7 +222,7 @@ homepage: http://msgpack.org/
|
|
228
222
|
licenses:
|
229
223
|
- Apache 2.0
|
230
224
|
metadata: {}
|
231
|
-
post_install_message:
|
225
|
+
post_install_message:
|
232
226
|
rdoc_options: []
|
233
227
|
require_paths:
|
234
228
|
- lib
|
@@ -236,7 +230,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
236
230
|
requirements:
|
237
231
|
- - ">="
|
238
232
|
- !ruby/object:Gem::Version
|
239
|
-
version: '
|
233
|
+
version: '2.4'
|
240
234
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
241
235
|
requirements:
|
242
236
|
- - ">="
|
@@ -244,31 +238,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
244
238
|
version: '0'
|
245
239
|
requirements: []
|
246
240
|
rubygems_version: 3.1.2
|
247
|
-
signing_key:
|
241
|
+
signing_key:
|
248
242
|
specification_version: 4
|
249
243
|
summary: MessagePack, a binary-based efficient data interchange format.
|
250
|
-
test_files:
|
251
|
-
- spec/cases.json
|
252
|
-
- spec/cases.msg
|
253
|
-
- spec/cases_compact.msg
|
254
|
-
- spec/cases_spec.rb
|
255
|
-
- spec/cruby/buffer_io_spec.rb
|
256
|
-
- spec/cruby/buffer_packer.rb
|
257
|
-
- spec/cruby/buffer_spec.rb
|
258
|
-
- spec/cruby/buffer_unpacker.rb
|
259
|
-
- spec/cruby/unpacker_spec.rb
|
260
|
-
- spec/ext_value_spec.rb
|
261
|
-
- spec/exttypes.rb
|
262
|
-
- spec/factory_spec.rb
|
263
|
-
- spec/format_spec.rb
|
264
|
-
- spec/jruby/benchmarks/shootout_bm.rb
|
265
|
-
- spec/jruby/benchmarks/symbolize_keys_bm.rb
|
266
|
-
- spec/jruby/unpacker_spec.rb
|
267
|
-
- spec/msgpack_spec.rb
|
268
|
-
- spec/pack_spec.rb
|
269
|
-
- spec/packer_spec.rb
|
270
|
-
- spec/random_compat.rb
|
271
|
-
- spec/spec_helper.rb
|
272
|
-
- spec/timestamp_spec.rb
|
273
|
-
- spec/unpack_spec.rb
|
274
|
-
- spec/unpacker_spec.rb
|
244
|
+
test_files: []
|
data/.travis.yml
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
|
3
|
-
sudo: false
|
4
|
-
|
5
|
-
branches:
|
6
|
-
only:
|
7
|
-
- master
|
8
|
-
|
9
|
-
gemfile:
|
10
|
-
- Gemfile
|
11
|
-
|
12
|
-
before_install:
|
13
|
-
# This is only for Ruby 2.5.0, Bundler 1.16.1 and rubygems 2.7.3
|
14
|
-
# See https://github.com/travis-ci/travis-ci/issues/8969
|
15
|
-
- "[ \"x2.7.3\" = \"x\"$(gem --version) ] && gem update --system --no-document || echo \"no need to update rubygem\""
|
16
|
-
|
17
|
-
# http://rubies.travis-ci.org/
|
18
|
-
matrix:
|
19
|
-
include:
|
20
|
-
- rvm: 2.3.8
|
21
|
-
os: linux
|
22
|
-
- rvm: 2.4.5
|
23
|
-
os: linux
|
24
|
-
- rvm: 2.5.3
|
25
|
-
os: linux
|
26
|
-
- rvm: 2.6.1
|
27
|
-
os: linux
|
28
|
-
- rvm: 2.6.1
|
29
|
-
os: osx
|
30
|
-
- rvm: ruby-head
|
31
|
-
os: linux
|
32
|
-
- rvm: jruby-9.1.9.0
|
33
|
-
os: linux
|
34
|
-
- rvm: jruby-head
|
35
|
-
os: linux
|
36
|
-
- rvm: jruby-19mode
|
37
|
-
os: linux
|
38
|
-
allow_failures:
|
39
|
-
- rvm: 2.6.1
|
40
|
-
os: osx
|
41
|
-
- rvm: ruby-head
|
42
|
-
- rvm: jruby-head
|
43
|
-
- rvm: jruby-19mode
|
data/README.rdoc
DELETED
@@ -1,225 +0,0 @@
|
|
1
|
-
|
2
|
-
= MessagePack
|
3
|
-
|
4
|
-
MessagePack[http://msgpack.org] is an efficient binary serialization format.
|
5
|
-
It lets you exchange data among multiple languages like JSON but it's faster and smaller.
|
6
|
-
For example, small integers (like flags or error code) are encoded into a single byte,
|
7
|
-
and typical short strings only require an extra byte in addition to the strings themselves.
|
8
|
-
|
9
|
-
If you ever wished to use JSON for convenience (storing an image with metadata) but could
|
10
|
-
not for technical reasons (binary data, size, speed...), MessagePack is a perfect replacement.
|
11
|
-
|
12
|
-
require 'msgpack'
|
13
|
-
msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03"
|
14
|
-
MessagePack.unpack(msg) #=> [1,2,3]
|
15
|
-
|
16
|
-
Use RubyGems to install:
|
17
|
-
|
18
|
-
gem install msgpack
|
19
|
-
|
20
|
-
or build msgpack-ruby and install:
|
21
|
-
|
22
|
-
bundle
|
23
|
-
rake
|
24
|
-
gem install --local pkg/msgpack
|
25
|
-
|
26
|
-
|
27
|
-
= Use cases
|
28
|
-
|
29
|
-
* Create REST API returing MessagePack using Rails + [RABL](https://github.com/nesquena/rabl)
|
30
|
-
* Store objects efficiently serialized by msgpack on memcached or Redis
|
31
|
-
* In fact Redis supports msgpack in EVAL-scripts[http://redis.io/commands/eval]
|
32
|
-
* Upload data in efficient format from mobile devices such as smartphones
|
33
|
-
* MessagePack works on iPhone/iPad and Android. See also Objective-C[https://github.com/msgpack/msgpack-objectivec] and Java[https://github.com/msgpack/msgpack-java] implementations
|
34
|
-
* Design a portable protocol to communicate with embedded devices
|
35
|
-
* Check also Fluentd[http://fluentd.org/] which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
|
36
|
-
* Exchange objects between software components written in different languages
|
37
|
-
* You'll need a flexible but efficient format so that components exchange objects while keeping compatibility
|
38
|
-
|
39
|
-
= Portability
|
40
|
-
|
41
|
-
MessagePack for Ruby should run on x86, ARM, PowerPC, SPARC and other CPU architectures.
|
42
|
-
|
43
|
-
And it works with MRI (CRuby) and Rubinius.
|
44
|
-
Patches to improve portability is highly welcomed.
|
45
|
-
|
46
|
-
|
47
|
-
= Serializing objects
|
48
|
-
|
49
|
-
Use *MessagePack.pack* or *to_msgpack*:
|
50
|
-
|
51
|
-
require 'msgpack'
|
52
|
-
msg = MessagePack.pack(obj) # or
|
53
|
-
msg = obj.to_msgpack
|
54
|
-
|
55
|
-
== Streaming serialization
|
56
|
-
|
57
|
-
Packer provides advanced API to serialize objects in streaming style:
|
58
|
-
|
59
|
-
# serialize a 2-element array [e1, e2]
|
60
|
-
pk = MessagePack::Packer.new(io)
|
61
|
-
pk.write_array_header(2).write(e1).write(e2).flush
|
62
|
-
|
63
|
-
See {API reference}[http://ruby.msgpack.org/MessagePack/Packer.html] for details.
|
64
|
-
|
65
|
-
= Deserializing objects
|
66
|
-
|
67
|
-
Use *MessagePack.unpack*:
|
68
|
-
|
69
|
-
require 'msgpack'
|
70
|
-
obj = MessagePack.unpack(msg)
|
71
|
-
|
72
|
-
== Streaming deserialization
|
73
|
-
|
74
|
-
Unpacker provides advanced API to deserialize objects in streaming style:
|
75
|
-
|
76
|
-
# deserialize objects from an IO
|
77
|
-
u = MessagePack::Unpacker.new(io)
|
78
|
-
u.each do |obj|
|
79
|
-
# ...
|
80
|
-
end
|
81
|
-
|
82
|
-
or event-driven style which works well with EventMachine:
|
83
|
-
|
84
|
-
# event-driven deserialization
|
85
|
-
def on_read(data)
|
86
|
-
@u ||= MessagePack::Unpacker.new
|
87
|
-
@u.feed_each(data) {|obj|
|
88
|
-
# ...
|
89
|
-
}
|
90
|
-
end
|
91
|
-
|
92
|
-
See {API reference}[http://ruby.msgpack.org/MessagePack/Unpacker.html] for details.
|
93
|
-
|
94
|
-
= Serializing and deserializing symbols
|
95
|
-
|
96
|
-
By default, symbols are serialized as strings:
|
97
|
-
|
98
|
-
packed = :symbol.to_msgpack # => "\xA6symbol"
|
99
|
-
MessagePack.unpack(packed) # => "symbol"
|
100
|
-
|
101
|
-
This can be customized by registering an extension type for them:
|
102
|
-
|
103
|
-
MessagePack::DefaultFactory.register_type(0x00, Symbol)
|
104
|
-
|
105
|
-
# symbols now survive round trips
|
106
|
-
packed = :symbol.to_msgpack # => "\xc7\x06\x00symbol"
|
107
|
-
MessagePack.unpack(packed) # => :symbol
|
108
|
-
|
109
|
-
The extension type for symbols is configurable like any other extension type.
|
110
|
-
For example, to customize how symbols are packed you can just redefine
|
111
|
-
Symbol#to_msgpack_ext. Doing this gives you an option to prevent symbols from
|
112
|
-
being serialized altogether by throwing an exception:
|
113
|
-
|
114
|
-
class Symbol
|
115
|
-
def to_msgpack_ext
|
116
|
-
raise "Serialization of symbols prohibited"
|
117
|
-
end
|
118
|
-
end
|
119
|
-
|
120
|
-
MessagePack::DefaultFactory.register_type(0x00, Symbol)
|
121
|
-
|
122
|
-
[1, :symbol, 'string'].to_msgpack # => RuntimeError: Serialization of symbols prohibited
|
123
|
-
|
124
|
-
= Serializing and deserializing Time instances
|
125
|
-
|
126
|
-
There are the timestamp extension type in MessagePack,
|
127
|
-
but it is not registered by default.
|
128
|
-
|
129
|
-
To map Ruby's Time to MessagePack's timestamp for the default factory:
|
130
|
-
|
131
|
-
MessagePack::DefaultFactory.register_type(
|
132
|
-
MessagePack::Timestamp::TYPE, # or just -1
|
133
|
-
Time,
|
134
|
-
packer: MessagePack::Time::Packer,
|
135
|
-
unpacker: MessagePack::Time::Unpacker
|
136
|
-
)
|
137
|
-
|
138
|
-
See {API reference}[http://ruby.msgpack.org/] for details.
|
139
|
-
|
140
|
-
= Extension Types
|
141
|
-
|
142
|
-
Packer and Unpacker support {Extension types of MessagePack}[https://github.com/msgpack/msgpack/blob/master/spec.md#types-extension-type].
|
143
|
-
|
144
|
-
# register how to serialize custom class at first
|
145
|
-
pk = MessagePack::Packer.new(io)
|
146
|
-
pk.register_type(0x01, MyClass1, :to_msgpack_ext) # equal to pk.register_type(0x01, MyClass)
|
147
|
-
pk.register_type(0x02, MyClass2){|obj| obj.how_to_serialize() } # blocks also available
|
148
|
-
|
149
|
-
# almost same API for unpacker
|
150
|
-
uk = MessagePack::Unpacker.new()
|
151
|
-
uk.register_type(0x01, MyClass1, :from_msgpack_ext)
|
152
|
-
uk.register_type(0x02){|data| MyClass2.create_from_serialized_data(data) }
|
153
|
-
|
154
|
-
MessagePack::Factory is to create packer and unpacker which have same extension types.
|
155
|
-
|
156
|
-
factory = MessagePack::Factory.new
|
157
|
-
factory.register_type(0x01, MyClass1) # same with next line
|
158
|
-
factory.register_type(0x01, MyClass1, packer: :to_msgpack_ext, unpacker: :from_msgpack_ext)
|
159
|
-
pk = factory.packer(options_for_packer)
|
160
|
-
uk = factory.unpacker(options_for_unpacker)
|
161
|
-
|
162
|
-
For *MessagePack.pack* and *MessagePack.unpack*, default packer/unpacker refer *MessagePack::DefaultFactory*. Call *MessagePack::DefaultFactory.register_type* to enable types process globally.
|
163
|
-
|
164
|
-
MessagePack::DefaultFactory.register_type(0x03, MyClass3)
|
165
|
-
MessagePack.unpack(data_with_ext_typeid_03) #=> MyClass3 instance
|
166
|
-
|
167
|
-
= Buffer API
|
168
|
-
|
169
|
-
MessagePack for Ruby provides a buffer API so that you can read or write data by hand, not via Packer or Unpacker API.
|
170
|
-
|
171
|
-
This {MessagePack::Buffer}[http://ruby.msgpack.org/MessagePack/Buffer.html] is backed with a fixed-length shared memory pool which is very fast for small data (<= 4KB),
|
172
|
-
and has zero-copy capability which significantly affects performance to handle large binary data.
|
173
|
-
|
174
|
-
= How to build and run tests
|
175
|
-
|
176
|
-
Before building msgpack, you need to install bundler and dependencies.
|
177
|
-
|
178
|
-
gem install bundler
|
179
|
-
bundle install
|
180
|
-
|
181
|
-
Then, you can run the tasks as follows:
|
182
|
-
|
183
|
-
* Build
|
184
|
-
|
185
|
-
bundle exec rake build
|
186
|
-
|
187
|
-
* Run tests
|
188
|
-
|
189
|
-
bundle exec rake spec
|
190
|
-
|
191
|
-
* Generating docs
|
192
|
-
|
193
|
-
bundle exec rake doc
|
194
|
-
|
195
|
-
== How to build -java rubygems
|
196
|
-
|
197
|
-
To build -java gems for JRuby, run:
|
198
|
-
|
199
|
-
rake build:java
|
200
|
-
|
201
|
-
If this directory has Gemfile.lock (generated with MRI), remove it beforehand.
|
202
|
-
|
203
|
-
== How to build -mingw32 rubygems
|
204
|
-
|
205
|
-
MessagePack mingw32/64 rubygems build process uses {rake-compiler-dock}[https://github.com/rake-compiler/rake-compiler-dock]. Run:
|
206
|
-
|
207
|
-
rake build:windows
|
208
|
-
|
209
|
-
Once this step successes, target gems exist in pkg/msgpack-*-{x86,x64}-mingw32.gem.
|
210
|
-
|
211
|
-
== Updating documents
|
212
|
-
|
213
|
-
Online documents (http://ruby.msgpack.org) is generated from gh-pages branch.
|
214
|
-
Following commands update documents in gh-pages branch:
|
215
|
-
|
216
|
-
bundle exec rake doc
|
217
|
-
git checkout gh-pages
|
218
|
-
cp doc/* ./ -a
|
219
|
-
|
220
|
-
= Copyright
|
221
|
-
|
222
|
-
Author:: Sadayuki Furuhashi <frsyuki@gmail.com>
|
223
|
-
Copyright:: Copyright (c) 2008-2015 Sadayuki Furuhashi
|
224
|
-
License:: Apache License, Version 2.0
|
225
|
-
|
data/bench/pack.rb
DELETED
@@ -1,23 +0,0 @@
|
|
1
|
-
require 'viiite'
|
2
|
-
require 'msgpack'
|
3
|
-
|
4
|
-
data = { 'hello' => 'world', 'nested' => ['structure', {value: 42}] }
|
5
|
-
data_sym = { hello: 'world', nested: ['structure', {value: 42}] }
|
6
|
-
|
7
|
-
data = MessagePack.pack(:hello => 'world', :nested => ['structure', {:value => 42}])
|
8
|
-
|
9
|
-
Viiite.bench do |b|
|
10
|
-
b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
|
11
|
-
b.report(:strings) do
|
12
|
-
runs.times do
|
13
|
-
MessagePack.pack(data)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
b.report(:symbols) do
|
18
|
-
runs.times do
|
19
|
-
MessagePack.pack(data_sym)
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
data/bench/pack_log.rb
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
require 'viiite'
|
2
|
-
require 'msgpack'
|
3
|
-
|
4
|
-
data_plain = { 'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"' }
|
5
|
-
data_structure = {
|
6
|
-
'remote_host' => '127.0.0.1',
|
7
|
-
'remote_user' => '-',
|
8
|
-
'date' => '10/Oct/2000:13:55:36 -0700',
|
9
|
-
'request' => 'GET /apache_pb.gif HTTP/1.0',
|
10
|
-
'method' => 'GET',
|
11
|
-
'path' => '/apache_pb.gif',
|
12
|
-
'protocol' => 'HTTP/1.0',
|
13
|
-
'status' => 200,
|
14
|
-
'bytes' => 2326,
|
15
|
-
'referer' => 'http://www.example.com/start.html',
|
16
|
-
'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
|
17
|
-
}
|
18
|
-
|
19
|
-
Viiite.bench do |b|
|
20
|
-
b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
|
21
|
-
b.report(:plain) do
|
22
|
-
runs.times do
|
23
|
-
MessagePack.pack(data_plain)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
b.report(:structure) do
|
28
|
-
runs.times do
|
29
|
-
MessagePack.pack(data_structure)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|