msgpack 1.3.3 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yaml +57 -0
- data/.rubocop.yml +2 -2
- data/ChangeLog +74 -0
- data/Gemfile +1 -1
- data/README.md +266 -0
- data/Rakefile +1 -9
- data/bench/bench.rb +78 -0
- data/bin/console +8 -0
- data/doclib/msgpack/factory.rb +47 -3
- data/doclib/msgpack/packer.rb +5 -4
- data/doclib/msgpack/unpacker.rb +2 -2
- data/ext/java/org/msgpack/jruby/Buffer.java +23 -16
- data/ext/java/org/msgpack/jruby/Decoder.java +46 -23
- data/ext/java/org/msgpack/jruby/Encoder.java +68 -30
- data/ext/java/org/msgpack/jruby/ExtensionRegistry.java +37 -49
- data/ext/java/org/msgpack/jruby/ExtensionValue.java +5 -8
- data/ext/java/org/msgpack/jruby/Factory.java +47 -7
- data/ext/java/org/msgpack/jruby/Packer.java +29 -17
- data/ext/java/org/msgpack/jruby/Unpacker.java +72 -37
- data/ext/msgpack/buffer.c +42 -68
- data/ext/msgpack/buffer.h +59 -14
- data/ext/msgpack/buffer_class.c +90 -52
- data/ext/msgpack/compat.h +1 -111
- data/ext/msgpack/extconf.rb +45 -19
- data/ext/msgpack/factory_class.c +133 -43
- data/ext/msgpack/packer.c +60 -36
- data/ext/msgpack/packer.h +27 -25
- data/ext/msgpack/packer_class.c +84 -77
- data/ext/msgpack/packer_class.h +11 -0
- data/ext/msgpack/packer_ext_registry.c +24 -32
- data/ext/msgpack/packer_ext_registry.h +40 -33
- data/ext/msgpack/sysdep.h +5 -2
- data/ext/msgpack/unpacker.c +132 -115
- data/ext/msgpack/unpacker.h +23 -10
- data/ext/msgpack/unpacker_class.c +83 -78
- data/ext/msgpack/unpacker_class.h +11 -0
- data/ext/msgpack/unpacker_ext_registry.c +42 -18
- data/ext/msgpack/unpacker_ext_registry.h +23 -16
- data/lib/msgpack/bigint.rb +69 -0
- data/lib/msgpack/factory.rb +103 -0
- data/lib/msgpack/symbol.rb +21 -4
- data/lib/msgpack/time.rb +1 -1
- data/lib/msgpack/version.rb +4 -8
- data/lib/msgpack.rb +6 -12
- data/msgpack.gemspec +4 -6
- data/spec/bigint_spec.rb +26 -0
- data/spec/cruby/buffer_spec.rb +17 -0
- data/spec/factory_spec.rb +351 -12
- data/spec/msgpack_spec.rb +1 -1
- data/spec/packer_spec.rb +18 -0
- data/spec/spec_helper.rb +37 -3
- data/spec/timestamp_spec.rb +38 -0
- data/spec/unpacker_spec.rb +157 -4
- metadata +31 -61
- data/.travis.yml +0 -43
- data/README.rdoc +0 -225
- data/bench/pack.rb +0 -23
- data/bench/pack_log.rb +0 -33
- data/bench/pack_log_long.rb +0 -65
- data/bench/pack_symbols.rb +0 -28
- data/bench/run.sh +0 -14
- data/bench/run_long.sh +0 -35
- data/bench/run_symbols.sh +0 -26
- data/bench/unpack.rb +0 -21
- data/bench/unpack_log.rb +0 -34
- data/bench/unpack_log_long.rb +0 -67
data/spec/unpacker_spec.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: ascii-8bit
|
2
|
-
|
3
1
|
require 'stringio'
|
4
2
|
require 'tempfile'
|
5
3
|
require 'zlib'
|
@@ -18,13 +16,35 @@ describe MessagePack::Unpacker do
|
|
18
16
|
it 'gets options to specify how to unpack values' do
|
19
17
|
u1 = MessagePack::Unpacker.new
|
20
18
|
u1.symbolize_keys?.should == false
|
19
|
+
u1.freeze?.should == false
|
21
20
|
u1.allow_unknown_ext?.should == false
|
22
21
|
|
23
|
-
u2 = MessagePack::Unpacker.new(symbolize_keys: true, allow_unknown_ext: true)
|
22
|
+
u2 = MessagePack::Unpacker.new(symbolize_keys: true, freeze: true, allow_unknown_ext: true)
|
24
23
|
u2.symbolize_keys?.should == true
|
24
|
+
u2.freeze?.should == true
|
25
25
|
u2.allow_unknown_ext?.should == true
|
26
26
|
end
|
27
27
|
|
28
|
+
if automatic_string_keys_deduplication?
|
29
|
+
it 'ensure string hash keys are deduplicated' do
|
30
|
+
sample_data = [{"foo" => 1}, {"foo" => 2}]
|
31
|
+
sample_packed = MessagePack.pack(sample_data).force_encoding('ASCII-8BIT')
|
32
|
+
unpacker.feed(sample_packed)
|
33
|
+
hashes = nil
|
34
|
+
unpacker.each { |obj| hashes = obj }
|
35
|
+
expect(hashes[0].keys.first).to equal(hashes[1].keys.first)
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'ensure strings are not deduplicated' do
|
39
|
+
sample_data = ["foo"]
|
40
|
+
sample_packed = MessagePack.pack(sample_data).force_encoding('ASCII-8BIT')
|
41
|
+
unpacker.feed(sample_packed)
|
42
|
+
ary = nil
|
43
|
+
unpacker.each { |obj| ary = obj }
|
44
|
+
expect(ary.first.frozen?).to eq(false)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
28
48
|
it 'gets IO or object which has #read to read data from it' do
|
29
49
|
sample_data = {"message" => "morning!", "num" => 1}
|
30
50
|
sample_packed = MessagePack.pack(sample_data).force_encoding('ASCII-8BIT')
|
@@ -280,6 +300,21 @@ describe MessagePack::Unpacker do
|
|
280
300
|
MessagePack.unpack(MessagePack.pack(symbolized_hash), :symbolize_keys => true).should == symbolized_hash
|
281
301
|
end
|
282
302
|
|
303
|
+
it 'MessagePack.unpack symbolize_keys preserve encoding' do
|
304
|
+
hash = { :ascii => 1, :utf8_é => 2}
|
305
|
+
loaded_hash = MessagePack.load(MessagePack.pack(hash), :symbolize_keys => true)
|
306
|
+
|
307
|
+
hash.keys[0].encoding.should == Encoding::US_ASCII # Ruby coerce symbols to US-ASCII when possible.
|
308
|
+
loaded_hash.keys[0].should == hash.keys[0]
|
309
|
+
loaded_hash.keys[0].encoding.should == hash.keys[0].encoding
|
310
|
+
|
311
|
+
hash.keys[1].encoding.should == Encoding::UTF_8
|
312
|
+
loaded_hash.keys[1].should == hash.keys[1]
|
313
|
+
loaded_hash.keys[1].encoding.should == hash.keys[1].encoding
|
314
|
+
|
315
|
+
MessagePack.unpack(MessagePack.pack(hash), :symbolize_keys => true).should == hash
|
316
|
+
end
|
317
|
+
|
283
318
|
it 'Unpacker#unpack symbolize_keys' do
|
284
319
|
unpacker = MessagePack::Unpacker.new(:symbolize_keys => true)
|
285
320
|
symbolized_hash = {:a => 'b', :c => 'd'}
|
@@ -621,6 +656,24 @@ describe MessagePack::Unpacker do
|
|
621
656
|
array = ['foo'] * 10_000
|
622
657
|
MessagePack.unpack(MessagePack.pack(array)).size.should == 10_000
|
623
658
|
end
|
659
|
+
|
660
|
+
it 'preserves string encoding (issue #200)' do
|
661
|
+
string = 'a'.force_encoding(Encoding::UTF_8)
|
662
|
+
MessagePack.unpack(MessagePack.pack(string)).encoding.should == string.encoding
|
663
|
+
|
664
|
+
string *= 256
|
665
|
+
MessagePack.unpack(MessagePack.pack(string)).encoding.should == string.encoding
|
666
|
+
end
|
667
|
+
|
668
|
+
it 'returns correct size for array16 (issue #127)' do
|
669
|
+
unpacker.feed("\xdc\x00\x01\x01")
|
670
|
+
unpacker.read_array_header.should == 1
|
671
|
+
end
|
672
|
+
|
673
|
+
it 'returns correct size for map16 (issue #127)' do
|
674
|
+
unpacker.feed("\xde\x00\x02\x01\x02\x03\x04")
|
675
|
+
unpacker.read_map_header.should == 2
|
676
|
+
end
|
624
677
|
end
|
625
678
|
|
626
679
|
context 'extensions' do
|
@@ -651,9 +704,109 @@ describe MessagePack::Unpacker do
|
|
651
704
|
end
|
652
705
|
end
|
653
706
|
|
707
|
+
context 'freeze' do
|
708
|
+
let :struct do
|
709
|
+
{'hello' => 'world', 'nested' => ['object', {'structure' => true}]}
|
710
|
+
end
|
711
|
+
|
712
|
+
let :buffer do
|
713
|
+
MessagePack.pack(struct)
|
714
|
+
end
|
715
|
+
|
716
|
+
let :unpacker do
|
717
|
+
described_class.new(:freeze => true)
|
718
|
+
end
|
719
|
+
|
720
|
+
if (-"test").equal?(-"test") # RUBY_VERSION >= "2.5"
|
721
|
+
it 'dedups strings' do
|
722
|
+
interned_str = -"test"
|
723
|
+
roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
|
724
|
+
expect(roundtrip).to be interned_str
|
725
|
+
|
726
|
+
interned_str = -""
|
727
|
+
roundtrip = MessagePack.unpack(MessagePack.pack(interned_str), freeze: true)
|
728
|
+
expect(roundtrip).to be interned_str
|
729
|
+
end
|
730
|
+
end
|
731
|
+
|
732
|
+
it 'can freeze objects when using .unpack' do
|
733
|
+
parsed_struct = MessagePack.unpack(buffer, freeze: true)
|
734
|
+
parsed_struct.should == struct
|
735
|
+
|
736
|
+
parsed_struct.should be_frozen
|
737
|
+
parsed_struct['hello'].should be_frozen
|
738
|
+
parsed_struct['nested'].should be_frozen
|
739
|
+
parsed_struct['nested'][0].should be_frozen
|
740
|
+
parsed_struct['nested'][1].should be_frozen
|
741
|
+
|
742
|
+
if string_deduplication?
|
743
|
+
parsed_struct.keys[0].should be_equal('hello'.freeze)
|
744
|
+
parsed_struct.keys[1].should be_equal('nested'.freeze)
|
745
|
+
parsed_struct.values[0].should be_equal('world'.freeze)
|
746
|
+
parsed_struct.values[1][0].should be_equal('object'.freeze)
|
747
|
+
parsed_struct.values[1][1].keys[0].should be_equal('structure'.freeze)
|
748
|
+
end
|
749
|
+
end
|
750
|
+
|
751
|
+
it 'can freeze objects when using #each' do
|
752
|
+
objs = []
|
753
|
+
unpacker.feed(buffer)
|
754
|
+
unpacker.each do |obj|
|
755
|
+
objs << obj
|
756
|
+
end
|
757
|
+
|
758
|
+
parsed_struct = objs.first
|
759
|
+
parsed_struct.should == struct
|
760
|
+
|
761
|
+
parsed_struct.should be_frozen
|
762
|
+
parsed_struct['hello'].should be_frozen
|
763
|
+
parsed_struct['nested'].should be_frozen
|
764
|
+
parsed_struct['nested'][0].should be_frozen
|
765
|
+
parsed_struct['nested'][1].should be_frozen
|
766
|
+
|
767
|
+
if string_deduplication?
|
768
|
+
parsed_struct.keys[0].should be_equal('hello'.freeze)
|
769
|
+
parsed_struct.keys[1].should be_equal('nested'.freeze)
|
770
|
+
parsed_struct.values[0].should be_equal('world'.freeze)
|
771
|
+
parsed_struct.values[1][0].should be_equal('object'.freeze)
|
772
|
+
parsed_struct.values[1][1].keys[0].should be_equal('structure'.freeze)
|
773
|
+
end
|
774
|
+
end
|
775
|
+
|
776
|
+
it 'can freeze objects when using #feed_each' do
|
777
|
+
objs = []
|
778
|
+
unpacker.feed_each(buffer) do |obj|
|
779
|
+
objs << obj
|
780
|
+
end
|
781
|
+
|
782
|
+
parsed_struct = objs.first
|
783
|
+
parsed_struct.should == struct
|
784
|
+
|
785
|
+
parsed_struct.should be_frozen
|
786
|
+
parsed_struct['hello'].should be_frozen
|
787
|
+
parsed_struct['nested'].should be_frozen
|
788
|
+
parsed_struct['nested'][0].should be_frozen
|
789
|
+
parsed_struct['nested'][1].should be_frozen
|
790
|
+
|
791
|
+
if string_deduplication?
|
792
|
+
parsed_struct.keys[0].should be_equal('hello'.freeze)
|
793
|
+
parsed_struct.keys[1].should be_equal('nested'.freeze)
|
794
|
+
parsed_struct.values[0].should be_equal('world'.freeze)
|
795
|
+
parsed_struct.values[1][0].should be_equal('object'.freeze)
|
796
|
+
parsed_struct.values[1][1].keys[0].should be_equal('structure'.freeze)
|
797
|
+
end
|
798
|
+
end
|
799
|
+
end
|
800
|
+
|
654
801
|
context 'binary encoding', :encodings do
|
655
802
|
let :buffer do
|
656
|
-
MessagePack.pack({
|
803
|
+
MessagePack.pack({
|
804
|
+
'hello'.b => 'world'.b,
|
805
|
+
'nested'.b => [
|
806
|
+
'object'.b,
|
807
|
+
{'structure'.b => true},
|
808
|
+
]
|
809
|
+
})
|
657
810
|
end
|
658
811
|
|
659
812
|
let :unpacker do
|
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: msgpack
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
- Theo Hultberg
|
9
9
|
- Satoshi Tagomori
|
10
|
-
autorequire:
|
10
|
+
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2022-09-30 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|
@@ -44,30 +44,16 @@ dependencies:
|
|
44
44
|
name: rake-compiler
|
45
45
|
requirement: !ruby/object:Gem::Requirement
|
46
46
|
requirements:
|
47
|
-
- - "
|
48
|
-
- !ruby/object:Gem::Version
|
49
|
-
version: '1.0'
|
50
|
-
type: :development
|
51
|
-
prerelease: false
|
52
|
-
version_requirements: !ruby/object:Gem::Requirement
|
53
|
-
requirements:
|
54
|
-
- - "~>"
|
55
|
-
- !ruby/object:Gem::Version
|
56
|
-
version: '1.0'
|
57
|
-
- !ruby/object:Gem::Dependency
|
58
|
-
name: rake-compiler-dock
|
59
|
-
requirement: !ruby/object:Gem::Requirement
|
60
|
-
requirements:
|
61
|
-
- - "~>"
|
47
|
+
- - ">="
|
62
48
|
- !ruby/object:Gem::Version
|
63
|
-
version:
|
49
|
+
version: 1.1.9
|
64
50
|
type: :development
|
65
51
|
prerelease: false
|
66
52
|
version_requirements: !ruby/object:Gem::Requirement
|
67
53
|
requirements:
|
68
|
-
- - "
|
54
|
+
- - ">="
|
69
55
|
- !ruby/object:Gem::Version
|
70
|
-
version:
|
56
|
+
version: 1.1.9
|
71
57
|
- !ruby/object:Gem::Dependency
|
72
58
|
name: rspec
|
73
59
|
requirement: !ruby/object:Gem::Requirement
|
@@ -110,6 +96,20 @@ dependencies:
|
|
110
96
|
- - ">="
|
111
97
|
- !ruby/object:Gem::Version
|
112
98
|
version: '0'
|
99
|
+
- !ruby/object:Gem::Dependency
|
100
|
+
name: benchmark-ips
|
101
|
+
requirement: !ruby/object:Gem::Requirement
|
102
|
+
requirements:
|
103
|
+
- - "~>"
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
version: 2.10.0
|
106
|
+
type: :development
|
107
|
+
prerelease: false
|
108
|
+
version_requirements: !ruby/object:Gem::Requirement
|
109
|
+
requirements:
|
110
|
+
- - "~>"
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: 2.10.0
|
113
113
|
description: MessagePack is a binary-based efficient object serialization library.
|
114
114
|
It enables to exchange structured objects between many languages like JSON. But
|
115
115
|
unlike JSON, it is very fast and small.
|
@@ -122,25 +122,17 @@ extensions:
|
|
122
122
|
- ext/msgpack/extconf.rb
|
123
123
|
extra_rdoc_files: []
|
124
124
|
files:
|
125
|
+
- ".github/workflows/ci.yaml"
|
125
126
|
- ".gitignore"
|
126
127
|
- ".rubocop.yml"
|
127
|
-
- ".travis.yml"
|
128
128
|
- ChangeLog
|
129
129
|
- Gemfile
|
130
130
|
- LICENSE
|
131
|
-
- README.
|
131
|
+
- README.md
|
132
132
|
- Rakefile
|
133
133
|
- appveyor.yml
|
134
|
-
- bench/
|
135
|
-
-
|
136
|
-
- bench/pack_log_long.rb
|
137
|
-
- bench/pack_symbols.rb
|
138
|
-
- bench/run.sh
|
139
|
-
- bench/run_long.sh
|
140
|
-
- bench/run_symbols.sh
|
141
|
-
- bench/unpack.rb
|
142
|
-
- bench/unpack_log.rb
|
143
|
-
- bench/unpack_log_long.rb
|
134
|
+
- bench/bench.rb
|
135
|
+
- bin/console
|
144
136
|
- doclib/msgpack.rb
|
145
137
|
- doclib/msgpack/buffer.rb
|
146
138
|
- doclib/msgpack/core_ext.rb
|
@@ -190,6 +182,7 @@ files:
|
|
190
182
|
- ext/msgpack/unpacker_ext_registry.c
|
191
183
|
- ext/msgpack/unpacker_ext_registry.h
|
192
184
|
- lib/msgpack.rb
|
185
|
+
- lib/msgpack/bigint.rb
|
193
186
|
- lib/msgpack/core_ext.rb
|
194
187
|
- lib/msgpack/factory.rb
|
195
188
|
- lib/msgpack/packer.rb
|
@@ -200,6 +193,7 @@ files:
|
|
200
193
|
- lib/msgpack/version.rb
|
201
194
|
- msgpack.gemspec
|
202
195
|
- msgpack.org.md
|
196
|
+
- spec/bigint_spec.rb
|
203
197
|
- spec/cases.json
|
204
198
|
- spec/cases.msg
|
205
199
|
- spec/cases_compact.msg
|
@@ -228,7 +222,7 @@ homepage: http://msgpack.org/
|
|
228
222
|
licenses:
|
229
223
|
- Apache 2.0
|
230
224
|
metadata: {}
|
231
|
-
post_install_message:
|
225
|
+
post_install_message:
|
232
226
|
rdoc_options: []
|
233
227
|
require_paths:
|
234
228
|
- lib
|
@@ -236,7 +230,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
236
230
|
requirements:
|
237
231
|
- - ">="
|
238
232
|
- !ruby/object:Gem::Version
|
239
|
-
version: '
|
233
|
+
version: '2.4'
|
240
234
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
241
235
|
requirements:
|
242
236
|
- - ">="
|
@@ -244,31 +238,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
244
238
|
version: '0'
|
245
239
|
requirements: []
|
246
240
|
rubygems_version: 3.1.2
|
247
|
-
signing_key:
|
241
|
+
signing_key:
|
248
242
|
specification_version: 4
|
249
243
|
summary: MessagePack, a binary-based efficient data interchange format.
|
250
|
-
test_files:
|
251
|
-
- spec/cases.json
|
252
|
-
- spec/cases.msg
|
253
|
-
- spec/cases_compact.msg
|
254
|
-
- spec/cases_spec.rb
|
255
|
-
- spec/cruby/buffer_io_spec.rb
|
256
|
-
- spec/cruby/buffer_packer.rb
|
257
|
-
- spec/cruby/buffer_spec.rb
|
258
|
-
- spec/cruby/buffer_unpacker.rb
|
259
|
-
- spec/cruby/unpacker_spec.rb
|
260
|
-
- spec/ext_value_spec.rb
|
261
|
-
- spec/exttypes.rb
|
262
|
-
- spec/factory_spec.rb
|
263
|
-
- spec/format_spec.rb
|
264
|
-
- spec/jruby/benchmarks/shootout_bm.rb
|
265
|
-
- spec/jruby/benchmarks/symbolize_keys_bm.rb
|
266
|
-
- spec/jruby/unpacker_spec.rb
|
267
|
-
- spec/msgpack_spec.rb
|
268
|
-
- spec/pack_spec.rb
|
269
|
-
- spec/packer_spec.rb
|
270
|
-
- spec/random_compat.rb
|
271
|
-
- spec/spec_helper.rb
|
272
|
-
- spec/timestamp_spec.rb
|
273
|
-
- spec/unpack_spec.rb
|
274
|
-
- spec/unpacker_spec.rb
|
244
|
+
test_files: []
|
data/.travis.yml
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
|
3
|
-
sudo: false
|
4
|
-
|
5
|
-
branches:
|
6
|
-
only:
|
7
|
-
- master
|
8
|
-
|
9
|
-
gemfile:
|
10
|
-
- Gemfile
|
11
|
-
|
12
|
-
before_install:
|
13
|
-
# This is only for Ruby 2.5.0, Bundler 1.16.1 and rubygems 2.7.3
|
14
|
-
# See https://github.com/travis-ci/travis-ci/issues/8969
|
15
|
-
- "[ \"x2.7.3\" = \"x\"$(gem --version) ] && gem update --system --no-document || echo \"no need to update rubygem\""
|
16
|
-
|
17
|
-
# http://rubies.travis-ci.org/
|
18
|
-
matrix:
|
19
|
-
include:
|
20
|
-
- rvm: 2.3.8
|
21
|
-
os: linux
|
22
|
-
- rvm: 2.4.5
|
23
|
-
os: linux
|
24
|
-
- rvm: 2.5.3
|
25
|
-
os: linux
|
26
|
-
- rvm: 2.6.1
|
27
|
-
os: linux
|
28
|
-
- rvm: 2.6.1
|
29
|
-
os: osx
|
30
|
-
- rvm: ruby-head
|
31
|
-
os: linux
|
32
|
-
- rvm: jruby-9.1.9.0
|
33
|
-
os: linux
|
34
|
-
- rvm: jruby-head
|
35
|
-
os: linux
|
36
|
-
- rvm: jruby-19mode
|
37
|
-
os: linux
|
38
|
-
allow_failures:
|
39
|
-
- rvm: 2.6.1
|
40
|
-
os: osx
|
41
|
-
- rvm: ruby-head
|
42
|
-
- rvm: jruby-head
|
43
|
-
- rvm: jruby-19mode
|
data/README.rdoc
DELETED
@@ -1,225 +0,0 @@
|
|
1
|
-
|
2
|
-
= MessagePack
|
3
|
-
|
4
|
-
MessagePack[http://msgpack.org] is an efficient binary serialization format.
|
5
|
-
It lets you exchange data among multiple languages like JSON but it's faster and smaller.
|
6
|
-
For example, small integers (like flags or error code) are encoded into a single byte,
|
7
|
-
and typical short strings only require an extra byte in addition to the strings themselves.
|
8
|
-
|
9
|
-
If you ever wished to use JSON for convenience (storing an image with metadata) but could
|
10
|
-
not for technical reasons (binary data, size, speed...), MessagePack is a perfect replacement.
|
11
|
-
|
12
|
-
require 'msgpack'
|
13
|
-
msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03"
|
14
|
-
MessagePack.unpack(msg) #=> [1,2,3]
|
15
|
-
|
16
|
-
Use RubyGems to install:
|
17
|
-
|
18
|
-
gem install msgpack
|
19
|
-
|
20
|
-
or build msgpack-ruby and install:
|
21
|
-
|
22
|
-
bundle
|
23
|
-
rake
|
24
|
-
gem install --local pkg/msgpack
|
25
|
-
|
26
|
-
|
27
|
-
= Use cases
|
28
|
-
|
29
|
-
* Create REST API returing MessagePack using Rails + [RABL](https://github.com/nesquena/rabl)
|
30
|
-
* Store objects efficiently serialized by msgpack on memcached or Redis
|
31
|
-
* In fact Redis supports msgpack in EVAL-scripts[http://redis.io/commands/eval]
|
32
|
-
* Upload data in efficient format from mobile devices such as smartphones
|
33
|
-
* MessagePack works on iPhone/iPad and Android. See also Objective-C[https://github.com/msgpack/msgpack-objectivec] and Java[https://github.com/msgpack/msgpack-java] implementations
|
34
|
-
* Design a portable protocol to communicate with embedded devices
|
35
|
-
* Check also Fluentd[http://fluentd.org/] which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
|
36
|
-
* Exchange objects between software components written in different languages
|
37
|
-
* You'll need a flexible but efficient format so that components exchange objects while keeping compatibility
|
38
|
-
|
39
|
-
= Portability
|
40
|
-
|
41
|
-
MessagePack for Ruby should run on x86, ARM, PowerPC, SPARC and other CPU architectures.
|
42
|
-
|
43
|
-
And it works with MRI (CRuby) and Rubinius.
|
44
|
-
Patches to improve portability is highly welcomed.
|
45
|
-
|
46
|
-
|
47
|
-
= Serializing objects
|
48
|
-
|
49
|
-
Use *MessagePack.pack* or *to_msgpack*:
|
50
|
-
|
51
|
-
require 'msgpack'
|
52
|
-
msg = MessagePack.pack(obj) # or
|
53
|
-
msg = obj.to_msgpack
|
54
|
-
|
55
|
-
== Streaming serialization
|
56
|
-
|
57
|
-
Packer provides advanced API to serialize objects in streaming style:
|
58
|
-
|
59
|
-
# serialize a 2-element array [e1, e2]
|
60
|
-
pk = MessagePack::Packer.new(io)
|
61
|
-
pk.write_array_header(2).write(e1).write(e2).flush
|
62
|
-
|
63
|
-
See {API reference}[http://ruby.msgpack.org/MessagePack/Packer.html] for details.
|
64
|
-
|
65
|
-
= Deserializing objects
|
66
|
-
|
67
|
-
Use *MessagePack.unpack*:
|
68
|
-
|
69
|
-
require 'msgpack'
|
70
|
-
obj = MessagePack.unpack(msg)
|
71
|
-
|
72
|
-
== Streaming deserialization
|
73
|
-
|
74
|
-
Unpacker provides advanced API to deserialize objects in streaming style:
|
75
|
-
|
76
|
-
# deserialize objects from an IO
|
77
|
-
u = MessagePack::Unpacker.new(io)
|
78
|
-
u.each do |obj|
|
79
|
-
# ...
|
80
|
-
end
|
81
|
-
|
82
|
-
or event-driven style which works well with EventMachine:
|
83
|
-
|
84
|
-
# event-driven deserialization
|
85
|
-
def on_read(data)
|
86
|
-
@u ||= MessagePack::Unpacker.new
|
87
|
-
@u.feed_each(data) {|obj|
|
88
|
-
# ...
|
89
|
-
}
|
90
|
-
end
|
91
|
-
|
92
|
-
See {API reference}[http://ruby.msgpack.org/MessagePack/Unpacker.html] for details.
|
93
|
-
|
94
|
-
= Serializing and deserializing symbols
|
95
|
-
|
96
|
-
By default, symbols are serialized as strings:
|
97
|
-
|
98
|
-
packed = :symbol.to_msgpack # => "\xA6symbol"
|
99
|
-
MessagePack.unpack(packed) # => "symbol"
|
100
|
-
|
101
|
-
This can be customized by registering an extension type for them:
|
102
|
-
|
103
|
-
MessagePack::DefaultFactory.register_type(0x00, Symbol)
|
104
|
-
|
105
|
-
# symbols now survive round trips
|
106
|
-
packed = :symbol.to_msgpack # => "\xc7\x06\x00symbol"
|
107
|
-
MessagePack.unpack(packed) # => :symbol
|
108
|
-
|
109
|
-
The extension type for symbols is configurable like any other extension type.
|
110
|
-
For example, to customize how symbols are packed you can just redefine
|
111
|
-
Symbol#to_msgpack_ext. Doing this gives you an option to prevent symbols from
|
112
|
-
being serialized altogether by throwing an exception:
|
113
|
-
|
114
|
-
class Symbol
|
115
|
-
def to_msgpack_ext
|
116
|
-
raise "Serialization of symbols prohibited"
|
117
|
-
end
|
118
|
-
end
|
119
|
-
|
120
|
-
MessagePack::DefaultFactory.register_type(0x00, Symbol)
|
121
|
-
|
122
|
-
[1, :symbol, 'string'].to_msgpack # => RuntimeError: Serialization of symbols prohibited
|
123
|
-
|
124
|
-
= Serializing and deserializing Time instances
|
125
|
-
|
126
|
-
There are the timestamp extension type in MessagePack,
|
127
|
-
but it is not registered by default.
|
128
|
-
|
129
|
-
To map Ruby's Time to MessagePack's timestamp for the default factory:
|
130
|
-
|
131
|
-
MessagePack::DefaultFactory.register_type(
|
132
|
-
MessagePack::Timestamp::TYPE, # or just -1
|
133
|
-
Time,
|
134
|
-
packer: MessagePack::Time::Packer,
|
135
|
-
unpacker: MessagePack::Time::Unpacker
|
136
|
-
)
|
137
|
-
|
138
|
-
See {API reference}[http://ruby.msgpack.org/] for details.
|
139
|
-
|
140
|
-
= Extension Types
|
141
|
-
|
142
|
-
Packer and Unpacker support {Extension types of MessagePack}[https://github.com/msgpack/msgpack/blob/master/spec.md#types-extension-type].
|
143
|
-
|
144
|
-
# register how to serialize custom class at first
|
145
|
-
pk = MessagePack::Packer.new(io)
|
146
|
-
pk.register_type(0x01, MyClass1, :to_msgpack_ext) # equal to pk.register_type(0x01, MyClass)
|
147
|
-
pk.register_type(0x02, MyClass2){|obj| obj.how_to_serialize() } # blocks also available
|
148
|
-
|
149
|
-
# almost same API for unpacker
|
150
|
-
uk = MessagePack::Unpacker.new()
|
151
|
-
uk.register_type(0x01, MyClass1, :from_msgpack_ext)
|
152
|
-
uk.register_type(0x02){|data| MyClass2.create_from_serialized_data(data) }
|
153
|
-
|
154
|
-
MessagePack::Factory is to create packer and unpacker which have same extension types.
|
155
|
-
|
156
|
-
factory = MessagePack::Factory.new
|
157
|
-
factory.register_type(0x01, MyClass1) # same with next line
|
158
|
-
factory.register_type(0x01, MyClass1, packer: :to_msgpack_ext, unpacker: :from_msgpack_ext)
|
159
|
-
pk = factory.packer(options_for_packer)
|
160
|
-
uk = factory.unpacker(options_for_unpacker)
|
161
|
-
|
162
|
-
For *MessagePack.pack* and *MessagePack.unpack*, default packer/unpacker refer *MessagePack::DefaultFactory*. Call *MessagePack::DefaultFactory.register_type* to enable types process globally.
|
163
|
-
|
164
|
-
MessagePack::DefaultFactory.register_type(0x03, MyClass3)
|
165
|
-
MessagePack.unpack(data_with_ext_typeid_03) #=> MyClass3 instance
|
166
|
-
|
167
|
-
= Buffer API
|
168
|
-
|
169
|
-
MessagePack for Ruby provides a buffer API so that you can read or write data by hand, not via Packer or Unpacker API.
|
170
|
-
|
171
|
-
This {MessagePack::Buffer}[http://ruby.msgpack.org/MessagePack/Buffer.html] is backed with a fixed-length shared memory pool which is very fast for small data (<= 4KB),
|
172
|
-
and has zero-copy capability which significantly affects performance to handle large binary data.
|
173
|
-
|
174
|
-
= How to build and run tests
|
175
|
-
|
176
|
-
Before building msgpack, you need to install bundler and dependencies.
|
177
|
-
|
178
|
-
gem install bundler
|
179
|
-
bundle install
|
180
|
-
|
181
|
-
Then, you can run the tasks as follows:
|
182
|
-
|
183
|
-
* Build
|
184
|
-
|
185
|
-
bundle exec rake build
|
186
|
-
|
187
|
-
* Run tests
|
188
|
-
|
189
|
-
bundle exec rake spec
|
190
|
-
|
191
|
-
* Generating docs
|
192
|
-
|
193
|
-
bundle exec rake doc
|
194
|
-
|
195
|
-
== How to build -java rubygems
|
196
|
-
|
197
|
-
To build -java gems for JRuby, run:
|
198
|
-
|
199
|
-
rake build:java
|
200
|
-
|
201
|
-
If this directory has Gemfile.lock (generated with MRI), remove it beforehand.
|
202
|
-
|
203
|
-
== How to build -mingw32 rubygems
|
204
|
-
|
205
|
-
MessagePack mingw32/64 rubygems build process uses {rake-compiler-dock}[https://github.com/rake-compiler/rake-compiler-dock]. Run:
|
206
|
-
|
207
|
-
rake build:windows
|
208
|
-
|
209
|
-
Once this step successes, target gems exist in pkg/msgpack-*-{x86,x64}-mingw32.gem.
|
210
|
-
|
211
|
-
== Updating documents
|
212
|
-
|
213
|
-
Online documents (http://ruby.msgpack.org) is generated from gh-pages branch.
|
214
|
-
Following commands update documents in gh-pages branch:
|
215
|
-
|
216
|
-
bundle exec rake doc
|
217
|
-
git checkout gh-pages
|
218
|
-
cp doc/* ./ -a
|
219
|
-
|
220
|
-
= Copyright
|
221
|
-
|
222
|
-
Author:: Sadayuki Furuhashi <frsyuki@gmail.com>
|
223
|
-
Copyright:: Copyright (c) 2008-2015 Sadayuki Furuhashi
|
224
|
-
License:: Apache License, Version 2.0
|
225
|
-
|
data/bench/pack.rb
DELETED
@@ -1,23 +0,0 @@
|
|
1
|
-
require 'viiite'
|
2
|
-
require 'msgpack'
|
3
|
-
|
4
|
-
data = { 'hello' => 'world', 'nested' => ['structure', {value: 42}] }
|
5
|
-
data_sym = { hello: 'world', nested: ['structure', {value: 42}] }
|
6
|
-
|
7
|
-
data = MessagePack.pack(:hello => 'world', :nested => ['structure', {:value => 42}])
|
8
|
-
|
9
|
-
Viiite.bench do |b|
|
10
|
-
b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
|
11
|
-
b.report(:strings) do
|
12
|
-
runs.times do
|
13
|
-
MessagePack.pack(data)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
b.report(:symbols) do
|
18
|
-
runs.times do
|
19
|
-
MessagePack.pack(data_sym)
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
data/bench/pack_log.rb
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
require 'viiite'
|
2
|
-
require 'msgpack'
|
3
|
-
|
4
|
-
data_plain = { 'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"' }
|
5
|
-
data_structure = {
|
6
|
-
'remote_host' => '127.0.0.1',
|
7
|
-
'remote_user' => '-',
|
8
|
-
'date' => '10/Oct/2000:13:55:36 -0700',
|
9
|
-
'request' => 'GET /apache_pb.gif HTTP/1.0',
|
10
|
-
'method' => 'GET',
|
11
|
-
'path' => '/apache_pb.gif',
|
12
|
-
'protocol' => 'HTTP/1.0',
|
13
|
-
'status' => 200,
|
14
|
-
'bytes' => 2326,
|
15
|
-
'referer' => 'http://www.example.com/start.html',
|
16
|
-
'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
|
17
|
-
}
|
18
|
-
|
19
|
-
Viiite.bench do |b|
|
20
|
-
b.range_over([10_000, 100_000, 1000_000], :runs) do |runs|
|
21
|
-
b.report(:plain) do
|
22
|
-
runs.times do
|
23
|
-
MessagePack.pack(data_plain)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
b.report(:structure) do
|
28
|
-
runs.times do
|
29
|
-
MessagePack.pack(data_structure)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|