msgpack 1.6.1 → 1.7.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ChangeLog +19 -0
- data/README.md +27 -0
- data/ext/java/org/msgpack/jruby/Buffer.java +3 -3
- data/ext/java/org/msgpack/jruby/ExtensionRegistry.java +7 -11
- data/ext/java/org/msgpack/jruby/ExtensionValue.java +1 -1
- data/ext/java/org/msgpack/jruby/Factory.java +11 -50
- data/ext/java/org/msgpack/jruby/Packer.java +9 -24
- data/ext/java/org/msgpack/jruby/Unpacker.java +15 -32
- data/ext/msgpack/buffer.c +16 -17
- data/ext/msgpack/buffer.h +2 -8
- data/ext/msgpack/buffer_class.c +72 -1
- data/ext/msgpack/buffer_class.h +1 -0
- data/ext/msgpack/extconf.rb +16 -26
- data/ext/msgpack/factory_class.c +27 -61
- data/ext/msgpack/packer.c +12 -14
- data/ext/msgpack/packer.h +0 -4
- data/ext/msgpack/packer_class.c +16 -37
- data/ext/msgpack/packer_ext_registry.c +31 -28
- data/ext/msgpack/packer_ext_registry.h +10 -14
- data/ext/msgpack/rmem.c +3 -4
- data/ext/msgpack/unpacker.c +11 -25
- data/ext/msgpack/unpacker.h +0 -4
- data/ext/msgpack/unpacker_class.c +9 -42
- data/ext/msgpack/unpacker_ext_registry.c +4 -16
- data/ext/msgpack/unpacker_ext_registry.h +3 -7
- data/lib/msgpack/factory.rb +90 -63
- data/lib/msgpack/packer.rb +6 -1
- data/lib/msgpack/unpacker.rb +10 -1
- data/lib/msgpack/version.rb +1 -1
- data/msgpack.gemspec +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2f5b1af6b3a51f5ccc6bcf67c94c1fc6193b02fe01b123e2cfb06a6df9607116
|
4
|
+
data.tar.gz: cc057f24e1ffa4cdc3e331499eb04de4c2383b0657dcf0baeba08300fd20862e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3eb06321a534ca9b16e321cc4a71458532578dafe7967314a662223b1fbf4aa93449c98177fa982aa532ce3732ddda4a6d497704df0e9c874da07f378c73595c
|
7
|
+
data.tar.gz: 8e540755e3db9e21d7dfa4354854e8b0486f5a1bbf82c3994c6095022205f7873153d364df9310d8072c481de38ca2b4c3e088e4221c3451ceb9438312489419
|
data/ChangeLog
CHANGED
@@ -1,3 +1,22 @@
|
|
1
|
+
2023-07-18 1.7.2:
|
2
|
+
|
3
|
+
* Fix a potential GC bug when packing data using recursive extensions and buffers containing over 512KkiB of data (See #341).
|
4
|
+
* Fix a regression where feeding an empty string to an Unpacker would be considered like the end of the buffer.
|
5
|
+
|
6
|
+
2023-05-19 1.7.1:
|
7
|
+
|
8
|
+
* Fix JRuby 9.4 compatibility.
|
9
|
+
* Fix compilation on older compilers (gcc 4.x).
|
10
|
+
* Fix an infinite recursion issue when registering a Symbol type with a `nil` packer.
|
11
|
+
|
12
|
+
2023-03-29 1.7.0:
|
13
|
+
|
14
|
+
* Fix a possible double-free issue when GC triggers inside `_msgpack_rmem_alloc2`.
|
15
|
+
* `Unpacker#feed` now always directly read in provided strings instead of copying content in its buffer.
|
16
|
+
* `Unpacker#feed` is now an alias of `Unpacker#feed_reference`.
|
17
|
+
* Implement `Factory::Pool#unpacker` and `Factory::Pool#packer` to allow for more precise serialization.
|
18
|
+
* Require Ruby 2.5+.
|
19
|
+
|
1
20
|
2023-03-03 1.6.1:
|
2
21
|
|
3
22
|
* Undefine `#clone` and `#dup` on `MessagePack::Buffer`, `MessagePack::Packer` and `MessagePack::Unpacker`.
|
data/README.md
CHANGED
@@ -211,6 +211,33 @@ factory.register_type(
|
|
211
211
|
factory.load(factory.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
|
212
212
|
```
|
213
213
|
|
214
|
+
## Pooling
|
215
|
+
|
216
|
+
Creating `Packer` and `Unpacker` objects is expensive. For best performance it is preferable to re-use these objects.
|
217
|
+
|
218
|
+
`MessagePack::Factory#pool` makes that easier:
|
219
|
+
|
220
|
+
```ruby
|
221
|
+
factory = MessagePack::Factory.new
|
222
|
+
factory.register_type(
|
223
|
+
0x01,
|
224
|
+
Point,
|
225
|
+
packer: ->(point, packer) {
|
226
|
+
packer.write(point.x)
|
227
|
+
packer.write(point.y)
|
228
|
+
},
|
229
|
+
unpacker: ->(unpacker) {
|
230
|
+
x = unpacker.read
|
231
|
+
y = unpacker.read
|
232
|
+
Point.new(x, y)
|
233
|
+
},
|
234
|
+
recursive: true,
|
235
|
+
)
|
236
|
+
pool = factory.pool(5) # The pool size should match the number of threads expected to use the factory concurrently.
|
237
|
+
|
238
|
+
pool.load(pool.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
|
239
|
+
```
|
240
|
+
|
214
241
|
## Buffer API
|
215
242
|
|
216
243
|
MessagePack for Ruby provides a buffer API so that you can read or write data by hand, not via Packer or Unpacker API.
|
@@ -22,10 +22,10 @@ import org.jcodings.Encoding;
|
|
22
22
|
@JRubyClass(name="MessagePack::Buffer")
|
23
23
|
public class Buffer extends RubyObject {
|
24
24
|
private static final long serialVersionUID = 8441244627425629412L;
|
25
|
-
private IRubyObject io;
|
26
|
-
private ByteBuffer buffer;
|
25
|
+
private transient IRubyObject io;
|
26
|
+
private transient ByteBuffer buffer;
|
27
27
|
private boolean writeMode;
|
28
|
-
private Encoding binaryEncoding;
|
28
|
+
private transient Encoding binaryEncoding;
|
29
29
|
|
30
30
|
private static final int CACHE_LINE_SIZE = 64;
|
31
31
|
private static final int ARRAY_HEADER_SIZE = 24;
|
@@ -54,8 +54,8 @@ public class ExtensionRegistry {
|
|
54
54
|
return hash;
|
55
55
|
}
|
56
56
|
|
57
|
-
public void put(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject
|
58
|
-
ExtensionEntry entry = new ExtensionEntry(mod, typeId, recursive, packerProc,
|
57
|
+
public void put(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject unpackerProc) {
|
58
|
+
ExtensionEntry entry = new ExtensionEntry(mod, typeId, recursive, packerProc, unpackerProc);
|
59
59
|
extensionsByModule.put(mod, entry);
|
60
60
|
extensionsByTypeId[typeId + 128] = entry;
|
61
61
|
extensionsByAncestor.clear();
|
@@ -114,18 +114,14 @@ public class ExtensionRegistry {
|
|
114
114
|
private final int typeId;
|
115
115
|
private final boolean recursive;
|
116
116
|
private final IRubyObject packerProc;
|
117
|
-
private final IRubyObject packerArg;
|
118
117
|
private final IRubyObject unpackerProc;
|
119
|
-
private final IRubyObject unpackerArg;
|
120
118
|
|
121
|
-
public ExtensionEntry(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject
|
119
|
+
public ExtensionEntry(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject unpackerProc) {
|
122
120
|
this.mod = mod;
|
123
121
|
this.typeId = typeId;
|
124
122
|
this.recursive = recursive;
|
125
123
|
this.packerProc = packerProc;
|
126
|
-
this.packerArg = packerArg;
|
127
124
|
this.unpackerProc = unpackerProc;
|
128
|
-
this.unpackerArg = unpackerArg;
|
129
125
|
}
|
130
126
|
|
131
127
|
public RubyModule getExtensionModule() {
|
@@ -141,11 +137,11 @@ public class ExtensionRegistry {
|
|
141
137
|
}
|
142
138
|
|
143
139
|
public boolean hasPacker() {
|
144
|
-
return packerProc != null;
|
140
|
+
return packerProc != null && !packerProc.isNil();
|
145
141
|
}
|
146
142
|
|
147
143
|
public boolean hasUnpacker() {
|
148
|
-
return unpackerProc != null;
|
144
|
+
return unpackerProc != null && !unpackerProc.isNil();
|
149
145
|
}
|
150
146
|
|
151
147
|
public IRubyObject getPackerProc() {
|
@@ -157,11 +153,11 @@ public class ExtensionRegistry {
|
|
157
153
|
}
|
158
154
|
|
159
155
|
public RubyArray<?> toPackerTuple(ThreadContext ctx) {
|
160
|
-
return ctx.runtime.newArray(new IRubyObject[] {ctx.runtime.newFixnum(typeId), packerProc
|
156
|
+
return ctx.runtime.newArray(new IRubyObject[] {ctx.runtime.newFixnum(typeId), packerProc});
|
161
157
|
}
|
162
158
|
|
163
159
|
public RubyArray<?> toUnpackerTuple(ThreadContext ctx) {
|
164
|
-
return ctx.runtime.newArray(new IRubyObject[] {mod, unpackerProc
|
160
|
+
return ctx.runtime.newArray(new IRubyObject[] {mod, unpackerProc});
|
165
161
|
}
|
166
162
|
|
167
163
|
public IRubyObject[] toPackerProcTypeIdPair(ThreadContext ctx) {
|
@@ -26,7 +26,7 @@ import static org.msgpack.jruby.Types.*;
|
|
26
26
|
@JRubyClass(name="MessagePack::ExtensionValue")
|
27
27
|
public class ExtensionValue extends RubyObject {
|
28
28
|
private static final long serialVersionUID = 8451274621449322492L;
|
29
|
-
private final Encoding binaryEncoding;
|
29
|
+
private transient final Encoding binaryEncoding;
|
30
30
|
|
31
31
|
private RubyFixnum type;
|
32
32
|
private RubyString payload;
|
@@ -26,8 +26,8 @@ import static org.jruby.runtime.Visibility.PRIVATE;
|
|
26
26
|
@JRubyClass(name="MessagePack::Factory")
|
27
27
|
public class Factory extends RubyObject {
|
28
28
|
private static final long serialVersionUID = 8441284623445322492L;
|
29
|
-
private final Ruby runtime;
|
30
|
-
private ExtensionRegistry extensionRegistry;
|
29
|
+
private transient final Ruby runtime;
|
30
|
+
private transient ExtensionRegistry extensionRegistry;
|
31
31
|
private boolean hasSymbolExtType;
|
32
32
|
private boolean hasBigIntExtType;
|
33
33
|
|
@@ -80,39 +80,15 @@ public class Factory extends RubyObject {
|
|
80
80
|
});
|
81
81
|
}
|
82
82
|
|
83
|
-
@JRubyMethod(name = "
|
84
|
-
public IRubyObject
|
85
|
-
|
86
|
-
IRubyObject type = args[0];
|
87
|
-
IRubyObject mod = args[1];
|
88
|
-
|
89
|
-
IRubyObject packerArg;
|
90
|
-
IRubyObject unpackerArg;
|
91
|
-
|
92
|
-
RubyHash options = null;
|
83
|
+
@JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
|
84
|
+
public IRubyObject registerTypeInternal(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject opts) {
|
85
|
+
testFrozen("MessagePack::Factory");
|
93
86
|
|
94
|
-
|
95
|
-
|
96
|
-
}
|
87
|
+
Ruby runtime = ctx.runtime;
|
88
|
+
RubyHash options = (RubyHash) opts;
|
97
89
|
|
98
|
-
|
99
|
-
|
100
|
-
unpackerArg = runtime.newSymbol("from_msgpack_ext");
|
101
|
-
} else if (args.length == 3) {
|
102
|
-
if (args[args.length - 1] instanceof RubyHash) {
|
103
|
-
options = (RubyHash) args[args.length - 1];
|
104
|
-
packerArg = options.fastARef(runtime.newSymbol("packer"));
|
105
|
-
unpackerArg = options.fastARef(runtime.newSymbol("unpacker"));
|
106
|
-
IRubyObject optimizedSymbolsParsingArg = options.fastARef(runtime.newSymbol("optimized_symbols_parsing"));
|
107
|
-
if (optimizedSymbolsParsingArg != null && optimizedSymbolsParsingArg.isTrue()) {
|
108
|
-
throw runtime.newArgumentError("JRuby implementation does not support the optimized_symbols_parsing option");
|
109
|
-
}
|
110
|
-
} else {
|
111
|
-
throw runtime.newArgumentError(String.format("expected Hash but found %s.", args[args.length - 1].getType().getName()));
|
112
|
-
}
|
113
|
-
} else {
|
114
|
-
throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 2..3)", 2 + args.length));
|
115
|
-
}
|
90
|
+
IRubyObject packerProc = options.fastARef(runtime.newSymbol("packer"));
|
91
|
+
IRubyObject unpackerProc = options.fastARef(runtime.newSymbol("unpacker"));
|
116
92
|
|
117
93
|
long typeId = ((RubyFixnum) type).getLongValue();
|
118
94
|
if (typeId < -128 || typeId > 127) {
|
@@ -124,21 +100,6 @@ public class Factory extends RubyObject {
|
|
124
100
|
}
|
125
101
|
RubyModule extModule = (RubyModule) mod;
|
126
102
|
|
127
|
-
IRubyObject packerProc = runtime.getNil();
|
128
|
-
IRubyObject unpackerProc = runtime.getNil();
|
129
|
-
if (packerArg != null) {
|
130
|
-
packerProc = packerArg.callMethod(ctx, "to_proc");
|
131
|
-
}
|
132
|
-
if (unpackerArg != null) {
|
133
|
-
if (unpackerArg instanceof RubyString || unpackerArg instanceof RubySymbol) {
|
134
|
-
unpackerProc = extModule.method(unpackerArg.callMethod(ctx, "to_sym"));
|
135
|
-
} else if (unpackerArg instanceof RubyProc || unpackerArg instanceof RubyMethod) {
|
136
|
-
unpackerProc = unpackerArg;
|
137
|
-
} else {
|
138
|
-
unpackerProc = unpackerArg.callMethod(ctx, "method", runtime.newSymbol("call"));
|
139
|
-
}
|
140
|
-
}
|
141
|
-
|
142
103
|
boolean recursive = false;
|
143
104
|
if (options != null) {
|
144
105
|
IRubyObject recursiveExtensionArg = options.fastARef(runtime.newSymbol("recursive"));
|
@@ -147,9 +108,9 @@ public class Factory extends RubyObject {
|
|
147
108
|
}
|
148
109
|
}
|
149
110
|
|
150
|
-
extensionRegistry.put(extModule, (int) typeId, recursive, packerProc,
|
111
|
+
extensionRegistry.put(extModule, (int) typeId, recursive, packerProc, unpackerProc);
|
151
112
|
|
152
|
-
if (extModule == runtime.getSymbol()) {
|
113
|
+
if (extModule == runtime.getSymbol() && !packerProc.isNil()) {
|
153
114
|
hasSymbolExtType = true;
|
154
115
|
}
|
155
116
|
|
@@ -28,12 +28,12 @@ import static org.jruby.runtime.Visibility.PRIVATE;
|
|
28
28
|
@JRubyClass(name="MessagePack::Packer")
|
29
29
|
public class Packer extends RubyObject {
|
30
30
|
private static final long serialVersionUID = 8451274621499362492L;
|
31
|
-
public ExtensionRegistry registry;
|
31
|
+
public transient ExtensionRegistry registry;
|
32
32
|
private Buffer buffer;
|
33
|
-
private Encoder encoder;
|
33
|
+
private transient Encoder encoder;
|
34
34
|
private boolean hasSymbolExtType;
|
35
35
|
private boolean hasBigintExtType;
|
36
|
-
private Encoding binaryEncoding;
|
36
|
+
private transient Encoding binaryEncoding;
|
37
37
|
|
38
38
|
public Packer(Ruby runtime, RubyClass type, ExtensionRegistry registry, boolean hasSymbolExtType, boolean hasBigintExtType) {
|
39
39
|
super(runtime, type);
|
@@ -93,26 +93,11 @@ public class Packer extends RubyObject {
|
|
93
93
|
return registry.toInternalPackerRegistry(ctx);
|
94
94
|
}
|
95
95
|
|
96
|
-
@JRubyMethod(name = "
|
97
|
-
public IRubyObject registerType(ThreadContext ctx, IRubyObject
|
96
|
+
@JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
|
97
|
+
public IRubyObject registerType(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject proc) {
|
98
|
+
testFrozen("MessagePack::Packer");
|
99
|
+
|
98
100
|
Ruby runtime = ctx.runtime;
|
99
|
-
IRubyObject type = args[0];
|
100
|
-
IRubyObject mod = args[1];
|
101
|
-
|
102
|
-
IRubyObject arg;
|
103
|
-
IRubyObject proc;
|
104
|
-
if (args.length == 2) {
|
105
|
-
if (! block.isGiven()) {
|
106
|
-
throw runtime.newLocalJumpErrorNoBlock();
|
107
|
-
}
|
108
|
-
proc = block.getProcObject();
|
109
|
-
arg = proc;
|
110
|
-
} else if (args.length == 3) {
|
111
|
-
arg = args[2];
|
112
|
-
proc = arg.callMethod(ctx, "to_proc");
|
113
|
-
} else {
|
114
|
-
throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 2..3)", 2 + args.length));
|
115
|
-
}
|
116
101
|
|
117
102
|
long typeId = ((RubyFixnum) type).getLongValue();
|
118
103
|
if (typeId < -128 || typeId > 127) {
|
@@ -124,9 +109,9 @@ public class Packer extends RubyObject {
|
|
124
109
|
}
|
125
110
|
RubyModule extModule = (RubyModule) mod;
|
126
111
|
|
127
|
-
registry.put(extModule, (int) typeId, false, proc,
|
112
|
+
registry.put(extModule, (int) typeId, false, proc, null);
|
128
113
|
|
129
|
-
if (extModule == runtime.getSymbol()) {
|
114
|
+
if (extModule == runtime.getSymbol() && !proc.isNil()) {
|
130
115
|
encoder.hasSymbolExtType = true;
|
131
116
|
}
|
132
117
|
|
@@ -21,18 +21,17 @@ import org.jruby.runtime.ThreadContext;
|
|
21
21
|
import org.jruby.anno.JRubyClass;
|
22
22
|
import org.jruby.anno.JRubyMethod;
|
23
23
|
import org.jruby.util.ByteList;
|
24
|
-
import org.jruby.ext.stringio.StringIO;
|
25
24
|
|
26
25
|
import static org.jruby.runtime.Visibility.PRIVATE;
|
27
26
|
|
28
27
|
@JRubyClass(name="MessagePack::Unpacker")
|
29
28
|
public class Unpacker extends RubyObject {
|
30
29
|
private static final long serialVersionUID = 8451264671199362492L;
|
31
|
-
private final ExtensionRegistry registry;
|
30
|
+
private transient final ExtensionRegistry registry;
|
32
31
|
|
33
|
-
private IRubyObject stream;
|
34
|
-
private IRubyObject data;
|
35
|
-
private Decoder decoder;
|
32
|
+
private transient IRubyObject stream;
|
33
|
+
private transient IRubyObject data;
|
34
|
+
private transient Decoder decoder;
|
36
35
|
private final RubyClass underflowErrorClass;
|
37
36
|
private boolean symbolizeKeys;
|
38
37
|
private boolean freeze;
|
@@ -127,37 +126,23 @@ public class Unpacker extends RubyObject {
|
|
127
126
|
return registry.toInternalUnpackerRegistry(ctx);
|
128
127
|
}
|
129
128
|
|
130
|
-
@JRubyMethod(name = "
|
131
|
-
public IRubyObject
|
129
|
+
@JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
|
130
|
+
public IRubyObject registerTypeInternal(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject proc) {
|
131
|
+
testFrozen("MessagePack::Unpacker");
|
132
|
+
|
132
133
|
Ruby runtime = ctx.runtime;
|
133
|
-
IRubyObject type = args[0];
|
134
|
-
|
135
|
-
RubyModule extModule;
|
136
|
-
IRubyObject arg;
|
137
|
-
IRubyObject proc;
|
138
|
-
if (args.length == 1) {
|
139
|
-
if (! block.isGiven()) {
|
140
|
-
throw runtime.newLocalJumpErrorNoBlock();
|
141
|
-
}
|
142
|
-
proc = RubyProc.newProc(runtime, block, block.type);
|
143
|
-
if (proc == null)
|
144
|
-
System.err.println("proc from Block is null");
|
145
|
-
arg = proc;
|
146
|
-
extModule = null;
|
147
|
-
} else if (args.length == 3) {
|
148
|
-
extModule = (RubyModule) args[1];
|
149
|
-
arg = args[2];
|
150
|
-
proc = extModule.method(arg);
|
151
|
-
} else {
|
152
|
-
throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 1 or 3)", 2 + args.length));
|
153
|
-
}
|
154
134
|
|
155
135
|
long typeId = ((RubyFixnum) type).getLongValue();
|
156
136
|
if (typeId < -128 || typeId > 127) {
|
157
137
|
throw runtime.newRangeError(String.format("integer %d too big to convert to `signed char'", typeId));
|
158
138
|
}
|
159
139
|
|
160
|
-
|
140
|
+
RubyModule extModule = null;
|
141
|
+
if (mod != runtime.getNil()) {
|
142
|
+
extModule = (RubyModule)mod;
|
143
|
+
}
|
144
|
+
|
145
|
+
registry.put(extModule, (int) typeId, false, null, proc);
|
161
146
|
return runtime.getNil();
|
162
147
|
}
|
163
148
|
|
@@ -331,9 +316,7 @@ public class Unpacker extends RubyObject {
|
|
331
316
|
@JRubyMethod(name = "stream=", required = 1)
|
332
317
|
public IRubyObject setStream(ThreadContext ctx, IRubyObject stream) {
|
333
318
|
RubyString str;
|
334
|
-
if (stream instanceof
|
335
|
-
str = stream.callMethod(ctx, "string").asString();
|
336
|
-
} else if (stream instanceof RubyIO) {
|
319
|
+
if (stream instanceof RubyIO) {
|
337
320
|
str = stream.callMethod(ctx, "read").asString();
|
338
321
|
} else if (stream.respondsTo("read")) {
|
339
322
|
str = stream.callMethod(ctx, "read").asString();
|
data/ext/msgpack/buffer.c
CHANGED
@@ -19,10 +19,6 @@
|
|
19
19
|
#include "buffer.h"
|
20
20
|
#include "rmem.h"
|
21
21
|
|
22
|
-
#ifndef HAVE_RB_STR_REPLACE
|
23
|
-
static ID s_replace;
|
24
|
-
#endif
|
25
|
-
|
26
22
|
int msgpack_rb_encindex_utf8;
|
27
23
|
int msgpack_rb_encindex_usascii;
|
28
24
|
int msgpack_rb_encindex_ascii8bit;
|
@@ -40,10 +36,6 @@ void msgpack_buffer_static_init(void)
|
|
40
36
|
msgpack_rb_encindex_ascii8bit = rb_ascii8bit_encindex();
|
41
37
|
|
42
38
|
msgpack_rmem_init(&s_rmem);
|
43
|
-
|
44
|
-
#ifndef HAVE_RB_STR_REPLACE
|
45
|
-
s_replace = rb_intern("replace");
|
46
|
-
#endif
|
47
39
|
}
|
48
40
|
|
49
41
|
void msgpack_buffer_static_destroy(void)
|
@@ -66,7 +58,11 @@ void msgpack_buffer_init(msgpack_buffer_t* b)
|
|
66
58
|
static void _msgpack_buffer_chunk_destroy(msgpack_buffer_chunk_t* c)
|
67
59
|
{
|
68
60
|
if(c->mem != NULL) {
|
69
|
-
if(
|
61
|
+
if(c->rmem) {
|
62
|
+
if(!msgpack_rmem_free(&s_rmem, c->mem)) {
|
63
|
+
rb_bug("Failed to free an rmem pointer, memory leak?");
|
64
|
+
}
|
65
|
+
} else {
|
70
66
|
xfree(c->mem);
|
71
67
|
}
|
72
68
|
/* no needs to update rmem_owner because chunks will not be
|
@@ -255,12 +251,14 @@ bool _msgpack_buffer_read_all2(msgpack_buffer_t* b, char* buffer, size_t length)
|
|
255
251
|
|
256
252
|
static inline msgpack_buffer_chunk_t* _msgpack_buffer_alloc_new_chunk(msgpack_buffer_t* b)
|
257
253
|
{
|
258
|
-
msgpack_buffer_chunk_t*
|
259
|
-
if(
|
260
|
-
|
254
|
+
msgpack_buffer_chunk_t* chunk = b->free_list;
|
255
|
+
if (chunk) {
|
256
|
+
b->free_list = b->free_list->next;
|
257
|
+
} else {
|
258
|
+
chunk = xmalloc(sizeof(msgpack_buffer_chunk_t));
|
261
259
|
}
|
262
|
-
|
263
|
-
return
|
260
|
+
memset(chunk, 0, sizeof(msgpack_buffer_chunk_t));
|
261
|
+
return chunk;
|
264
262
|
}
|
265
263
|
|
266
264
|
static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
|
@@ -330,14 +328,12 @@ static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE s
|
|
330
328
|
|
331
329
|
void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
|
332
330
|
{
|
333
|
-
size_t length = RSTRING_LEN(string);
|
334
|
-
|
335
331
|
if(b->io != Qnil) {
|
336
332
|
msgpack_buffer_flush(b);
|
337
333
|
if (ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit) {
|
338
334
|
rb_funcall(b->io, b->io_write_all_method, 1, string);
|
339
335
|
} else {
|
340
|
-
msgpack_buffer_append(b, RSTRING_PTR(string),
|
336
|
+
msgpack_buffer_append(b, RSTRING_PTR(string), RSTRING_LEN(string));
|
341
337
|
}
|
342
338
|
} else {
|
343
339
|
_msgpack_buffer_append_reference(b, string);
|
@@ -349,6 +345,8 @@ static inline void* _msgpack_buffer_chunk_malloc(
|
|
349
345
|
size_t required_size, size_t* allocated_size)
|
350
346
|
{
|
351
347
|
if(required_size <= MSGPACK_RMEM_PAGE_SIZE) {
|
348
|
+
c->rmem = true;
|
349
|
+
|
352
350
|
if((size_t)(b->rmem_end - b->rmem_last) < required_size) {
|
353
351
|
/* alloc new rmem page */
|
354
352
|
*allocated_size = MSGPACK_RMEM_PAGE_SIZE;
|
@@ -379,6 +377,7 @@ static inline void* _msgpack_buffer_chunk_malloc(
|
|
379
377
|
*allocated_size = required_size;
|
380
378
|
void* mem = xmalloc(required_size);
|
381
379
|
c->mem = mem;
|
380
|
+
c->rmem = false;
|
382
381
|
return mem;
|
383
382
|
}
|
384
383
|
|
data/ext/msgpack/buffer.h
CHANGED
@@ -78,6 +78,7 @@ struct msgpack_buffer_chunk_t {
|
|
78
78
|
void* mem;
|
79
79
|
msgpack_buffer_chunk_t* next;
|
80
80
|
VALUE mapped_string; /* RBString or NO_MAPPED_STRING */
|
81
|
+
bool rmem;
|
81
82
|
};
|
82
83
|
|
83
84
|
union msgpack_buffer_cast_block_t {
|
@@ -267,14 +268,9 @@ static inline size_t msgpack_buffer_append_string(msgpack_buffer_t* b, VALUE str
|
|
267
268
|
static inline size_t msgpack_buffer_append_string_reference(msgpack_buffer_t* b, VALUE string)
|
268
269
|
{
|
269
270
|
size_t length = RSTRING_LEN(string);
|
270
|
-
|
271
|
-
if(length > MSGPACK_BUFFER_STRING_WRITE_REFERENCE_MINIMUM) {
|
271
|
+
if (length > 0) {
|
272
272
|
_msgpack_buffer_append_long_string(b, string);
|
273
|
-
|
274
|
-
} else {
|
275
|
-
msgpack_buffer_append(b, RSTRING_PTR(string), length);
|
276
273
|
}
|
277
|
-
|
278
274
|
return length;
|
279
275
|
}
|
280
276
|
|
@@ -479,7 +475,6 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
|
|
479
475
|
result = rb_str_new(b->read_buffer, length);
|
480
476
|
}
|
481
477
|
|
482
|
-
#if STR_UMINUS_DEDUPE
|
483
478
|
if (will_be_frozen) {
|
484
479
|
#if STR_UMINUS_DEDUPE_FROZEN
|
485
480
|
// Starting from MRI 2.8 it is preferable to freeze the string
|
@@ -491,7 +486,6 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
|
|
491
486
|
// frozen.
|
492
487
|
result = rb_funcall(result, s_uminus, 0);
|
493
488
|
}
|
494
|
-
#endif // STR_UMINUS_DEDUPE
|
495
489
|
_msgpack_buffer_consumed(b, length);
|
496
490
|
return result;
|
497
491
|
|
data/ext/msgpack/buffer_class.c
CHANGED
@@ -21,7 +21,8 @@
|
|
21
21
|
#include "buffer.h"
|
22
22
|
#include "buffer_class.h"
|
23
23
|
|
24
|
-
VALUE cMessagePack_Buffer;
|
24
|
+
VALUE cMessagePack_Buffer = Qnil;
|
25
|
+
VALUE cMessagePack_HeldBuffer = Qnil;
|
25
26
|
|
26
27
|
static ID s_read;
|
27
28
|
static ID s_readpartial;
|
@@ -34,6 +35,73 @@ static VALUE sym_read_reference_threshold;
|
|
34
35
|
static VALUE sym_write_reference_threshold;
|
35
36
|
static VALUE sym_io_buffer_size;
|
36
37
|
|
38
|
+
typedef struct msgpack_held_buffer_t msgpack_held_buffer_t;
|
39
|
+
struct msgpack_held_buffer_t {
|
40
|
+
size_t size;
|
41
|
+
VALUE mapped_strings[];
|
42
|
+
};
|
43
|
+
|
44
|
+
static void HeldBuffer_mark(void *data)
|
45
|
+
{
|
46
|
+
msgpack_held_buffer_t* held_buffer = (msgpack_held_buffer_t*)data;
|
47
|
+
for (size_t index = 0; index < held_buffer->size; index++) {
|
48
|
+
rb_gc_mark(held_buffer->mapped_strings[index]);
|
49
|
+
}
|
50
|
+
}
|
51
|
+
|
52
|
+
static size_t HeldBuffer_memsize(const void *data)
|
53
|
+
{
|
54
|
+
const msgpack_held_buffer_t* held_buffer = (msgpack_held_buffer_t*)data;
|
55
|
+
return sizeof(size_t) + sizeof(VALUE) * held_buffer->size;
|
56
|
+
}
|
57
|
+
|
58
|
+
static const rb_data_type_t held_buffer_data_type = {
|
59
|
+
.wrap_struct_name = "msgpack:held_buffer",
|
60
|
+
.function = {
|
61
|
+
.dmark = HeldBuffer_mark,
|
62
|
+
.dfree = RUBY_TYPED_DEFAULT_FREE,
|
63
|
+
.dsize = HeldBuffer_memsize,
|
64
|
+
},
|
65
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY
|
66
|
+
};
|
67
|
+
|
68
|
+
VALUE MessagePack_Buffer_hold(msgpack_buffer_t* buffer)
|
69
|
+
{
|
70
|
+
size_t mapped_strings_count = 0;
|
71
|
+
msgpack_buffer_chunk_t* c = buffer->head;
|
72
|
+
while (c != &buffer->tail) {
|
73
|
+
if (c->mapped_string != NO_MAPPED_STRING) {
|
74
|
+
mapped_strings_count++;
|
75
|
+
}
|
76
|
+
c = c->next;
|
77
|
+
}
|
78
|
+
if (c->mapped_string != NO_MAPPED_STRING) {
|
79
|
+
mapped_strings_count++;
|
80
|
+
}
|
81
|
+
|
82
|
+
if (mapped_strings_count == 0) {
|
83
|
+
return Qnil;
|
84
|
+
}
|
85
|
+
|
86
|
+
msgpack_held_buffer_t* held_buffer = xmalloc(sizeof(msgpack_held_buffer_t) + mapped_strings_count * sizeof(VALUE));
|
87
|
+
|
88
|
+
c = buffer->head;
|
89
|
+
mapped_strings_count = 0;
|
90
|
+
while (c != &buffer->tail) {
|
91
|
+
if (c->mapped_string != NO_MAPPED_STRING) {
|
92
|
+
held_buffer->mapped_strings[mapped_strings_count] = c->mapped_string;
|
93
|
+
mapped_strings_count++;
|
94
|
+
}
|
95
|
+
c = c->next;
|
96
|
+
}
|
97
|
+
if (c->mapped_string != NO_MAPPED_STRING) {
|
98
|
+
held_buffer->mapped_strings[mapped_strings_count] = c->mapped_string;
|
99
|
+
mapped_strings_count++;
|
100
|
+
}
|
101
|
+
held_buffer->size = mapped_strings_count;
|
102
|
+
return TypedData_Wrap_Struct(cMessagePack_HeldBuffer, &held_buffer_data_type, held_buffer);
|
103
|
+
}
|
104
|
+
|
37
105
|
|
38
106
|
#define CHECK_STRING_TYPE(value) \
|
39
107
|
value = rb_check_string_type(value); \
|
@@ -520,6 +588,9 @@ void MessagePack_Buffer_module_init(VALUE mMessagePack)
|
|
520
588
|
|
521
589
|
msgpack_buffer_static_init();
|
522
590
|
|
591
|
+
cMessagePack_HeldBuffer = rb_define_class_under(mMessagePack, "HeldBuffer", rb_cBasicObject);
|
592
|
+
rb_undef_alloc_func(cMessagePack_HeldBuffer);
|
593
|
+
|
523
594
|
cMessagePack_Buffer = rb_define_class_under(mMessagePack, "Buffer", rb_cObject);
|
524
595
|
|
525
596
|
rb_define_alloc_func(cMessagePack_Buffer, Buffer_alloc);
|
data/ext/msgpack/buffer_class.h
CHANGED
@@ -25,6 +25,7 @@ extern VALUE cMessagePack_Buffer;
|
|
25
25
|
void MessagePack_Buffer_module_init(VALUE mMessagePack);
|
26
26
|
|
27
27
|
VALUE MessagePack_Buffer_wrap(msgpack_buffer_t* b, VALUE owner);
|
28
|
+
VALUE MessagePack_Buffer_hold(msgpack_buffer_t* b);
|
28
29
|
|
29
30
|
void MessagePack_Buffer_set_options(msgpack_buffer_t* b, VALUE io, VALUE options);
|
30
31
|
|