msgpack 1.5.1 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c5ae461c691018c21ea088c4b629510c0626a31c16e640d7128d1d4be70276a
-  data.tar.gz: 32ac330fedeb8fdcf90e172df4480d05564185ef9b3e72f430c83de04eea90b1
+  metadata.gz: 2f5b1af6b3a51f5ccc6bcf67c94c1fc6193b02fe01b123e2cfb06a6df9607116
+  data.tar.gz: cc057f24e1ffa4cdc3e331499eb04de4c2383b0657dcf0baeba08300fd20862e
 SHA512:
-  metadata.gz: 2a5a1c96cbfdc2030c38cf559a61aaacc466a5b4fcaaa6af809b8d062f3042a44ee6df0f731fe77bbdf1ebba0a123da2b9db033e7117baabcc6e9664dd6f56ef
-  data.tar.gz: 07c313bd70f92996b0131a0c5ce9836f025dd4fa98fdbe365ad04fa860b2bd6ca81c62fbd90cc2296a75eddd2a9b648b3ca48b8838a40ffab72a5845dfd6e724
+  metadata.gz: 3eb06321a534ca9b16e321cc4a71458532578dafe7967314a662223b1fbf4aa93449c98177fa982aa532ce3732ddda4a6d497704df0e9c874da07f378c73595c
+  data.tar.gz: 8e540755e3db9e21d7dfa4354854e8b0486f5a1bbf82c3994c6095022205f7873153d364df9310d8072c481de38ca2b4c3e088e4221c3451ceb9438312489419

data/ChangeLog

@@ -1,3 +1,58 @@
+2023-07-18 1.7.2:
+
+* Fix a potential GC bug when packing data using recursive extensions and buffers containing over 512KkiB of data (See #341).
+* Fix a regression where feeding an empty string to an Unpacker would be considered like the end of the buffer.
+
+2023-05-19 1.7.1:
+
+* Fix JRuby 9.4 compatibility.
+* Fix compilation on older compilers (gcc 4.x).
+* Fix an infinite recursion issue when registering a Symbol type with a `nil` packer.
+
+2023-03-29 1.7.0:
+
+* Fix a possible double-free issue when GC triggers inside `_msgpack_rmem_alloc2`.
+* `Unpacker#feed` now always directly read in provided strings instead of copying content in its buffer.
+* `Unpacker#feed` is now an alias of `Unpacker#feed_reference`.
+* Implement `Factory::Pool#unpacker` and `Factory::Pool#packer` to allow for more precise serialization.
+* Require Ruby 2.5+.
+
+2023-03-03 1.6.1:
+
+* Undefine `#clone` and `#dup` on `MessagePack::Buffer`, `MessagePack::Packer` and `MessagePack::Unpacker`.
+  These methods were never intended, and using them could cause leaks or crashes or worse.
+* Fix a possible GC crash when GC trigger inside `MessagePack::Buffer.new` (#314).
+
+2022-09-30 1.6.0:
+
+* Fix a potential use-after-free bug in Buffer_free when accessing a packer or unpacker buffer. 
+* `old-style-definition` compilation warnings.
+* Restore zero-copy buffer feed when provided a Ruby string. This was accidentally broken in 1.5.4.
+* Provide implementations for `ObjectSpace.memsize`. Message pack objects now properly report their size to Ruby.
+* Fix an endianess bug on Windows platform.
+
+2022-08-23 1.5.6:
+
+* No actual code change, just re-release the `java` version properly.
+
+2022-08-22 1.5.5:
+
+* Fix a segfault when GC triggers inside a recursive extension.
+
+2022-07-25 1.5.4:
+
+* Fix a segfault when deserializing empty symbol (`:""`).
+* Improve compilation flags to not strip debug symbols.
+
+2022-05-30 version 1.5.3:
+
+* Fix deduplication of empty strings when using the `freeze: true` option.
+* Use `rb_hash_new_capa` when available (Ruby 3.2) for improved performance when parsing large hashes.
+
+2022-05-27 version 1.5.2:
+
+* Fix bug about unpacking ext type objects with the recursive option
+
 2022-04-07 version 1.5.1:
 
 * Fix bug about packing/unpacking ext type objects with the recursive option

data/README.md

@@ -40,7 +40,7 @@ or build msgpack-ruby and install:
 MessagePack for Ruby should run on x86, ARM, PowerPC, SPARC and other CPU architectures.
 
 And it works with MRI (CRuby) and Rubinius.
-Patches to improve portability is highly welcomed.
+Patches to improve portability are highly welcomed.
 
 
 ## Serializing objects
@@ -51,6 +51,7 @@ Use `MessagePack.pack` or `to_msgpack`:
 require 'msgpack'
 msg = MessagePack.pack(obj)  # or
 msg = obj.to_msgpack
+File.binwrite('mydata.msgpack', msg)
 ```
 
 ### Streaming serialization
@@ -71,6 +72,7 @@ Use `MessagePack.unpack`:
 
 ```ruby
 require 'msgpack'
+msg = File.binread('mydata.msgpack')
 obj = MessagePack.unpack(msg)
 ```
 
@@ -209,6 +211,33 @@ factory.register_type(
 factory.load(factory.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
 ```
 
+## Pooling
+
+Creating `Packer` and `Unpacker` objects is expensive. For best performance it is preferable to re-use these objects.
+
+`MessagePack::Factory#pool` makes that easier:
+
+```ruby
+factory = MessagePack::Factory.new
+factory.register_type(
+  0x01,
+  Point,
+  packer: ->(point, packer) {
+    packer.write(point.x)
+    packer.write(point.y)
+  },
+  unpacker: ->(unpacker) {
+    x = unpacker.read
+    y = unpacker.read
+    Point.new(x, y)
+  },
+  recursive: true,
+)
+pool = factory.pool(5) # The pool size should match the number of threads expected to use the factory concurrently.
+
+pool.load(pool.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
+```
+
 ## Buffer API
 
 MessagePack for Ruby provides a buffer API so that you can read or write data by hand, not via Packer or Unpacker API.

data/ext/java/org/msgpack/jruby/Buffer.java

@@ -22,10 +22,10 @@ import org.jcodings.Encoding;
 @JRubyClass(name="MessagePack::Buffer")
 public class Buffer extends RubyObject {
   private static final long serialVersionUID = 8441244627425629412L;
-  private IRubyObject io;
-  private ByteBuffer buffer;
+  private transient IRubyObject io;
+  private transient ByteBuffer buffer;
   private boolean writeMode;
-  private Encoding binaryEncoding;
+  private transient Encoding binaryEncoding;
 
   private static final int CACHE_LINE_SIZE = 64;
   private static final int ARRAY_HEADER_SIZE = 24;

data/ext/java/org/msgpack/jruby/ExtensionRegistry.java

@@ -18,22 +18,17 @@ public class ExtensionRegistry {
   private final ExtensionEntry[] extensionsByTypeId;
 
   public ExtensionRegistry() {
-    this(new HashMap<RubyModule, ExtensionEntry>());
+    this(new HashMap<RubyModule, ExtensionEntry>(), new ExtensionEntry[256]);
   }
 
-  private ExtensionRegistry(Map<RubyModule, ExtensionEntry> extensionsByModule) {
+  private ExtensionRegistry(Map<RubyModule, ExtensionEntry> extensionsByModule, ExtensionEntry[] extensionsByTypeId) {
     this.extensionsByModule = new HashMap<RubyModule, ExtensionEntry>(extensionsByModule);
     this.extensionsByAncestor = new HashMap<RubyModule, ExtensionEntry>();
-    this.extensionsByTypeId = new ExtensionEntry[256];
-    for (ExtensionEntry entry : extensionsByModule.values()) {
-      if (entry.hasUnpacker()) {
-        extensionsByTypeId[entry.getTypeId() + 128] = entry;
-      }
-    }
+    this.extensionsByTypeId = extensionsByTypeId.clone();
   }
 
   public ExtensionRegistry dup() {
-    return new ExtensionRegistry(extensionsByModule);
+    return new ExtensionRegistry(extensionsByModule, extensionsByTypeId);
   }
 
   public IRubyObject toInternalPackerRegistry(ThreadContext ctx) {
@@ -59,8 +54,8 @@ public class ExtensionRegistry {
     return hash;
   }
 
-  public void put(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject packerArg, IRubyObject unpackerProc, IRubyObject unpackerArg) {
-    ExtensionEntry entry = new ExtensionEntry(mod, typeId, recursive, packerProc, packerArg, unpackerProc, unpackerArg);
+  public void put(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject unpackerProc) {
+    ExtensionEntry entry = new ExtensionEntry(mod, typeId, recursive, packerProc, unpackerProc);
     extensionsByModule.put(mod, entry);
     extensionsByTypeId[typeId + 128] = entry;
     extensionsByAncestor.clear();
@@ -119,18 +114,14 @@ public class ExtensionRegistry {
     private final int typeId;
     private final boolean recursive;
     private final IRubyObject packerProc;
-    private final IRubyObject packerArg;
     private final IRubyObject unpackerProc;
-    private final IRubyObject unpackerArg;
 
-    public ExtensionEntry(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject packerArg, IRubyObject unpackerProc, IRubyObject unpackerArg) {
+    public ExtensionEntry(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject unpackerProc) {
       this.mod = mod;
       this.typeId = typeId;
       this.recursive = recursive;
       this.packerProc = packerProc;
-      this.packerArg = packerArg;
       this.unpackerProc = unpackerProc;
-      this.unpackerArg = unpackerArg;
     }
 
     public RubyModule getExtensionModule() {
@@ -146,11 +137,11 @@ public class ExtensionRegistry {
     }
 
     public boolean hasPacker() {
-      return packerProc != null;
+      return packerProc != null && !packerProc.isNil();
     }
 
     public boolean hasUnpacker() {
-      return unpackerProc != null;
+      return unpackerProc != null && !unpackerProc.isNil();
     }
 
     public IRubyObject getPackerProc() {
@@ -162,11 +153,11 @@ public class ExtensionRegistry {
     }
 
     public RubyArray<?> toPackerTuple(ThreadContext ctx) {
-      return ctx.runtime.newArray(new IRubyObject[] {ctx.runtime.newFixnum(typeId), packerProc, packerArg});
+      return ctx.runtime.newArray(new IRubyObject[] {ctx.runtime.newFixnum(typeId), packerProc});
     }
 
     public RubyArray<?> toUnpackerTuple(ThreadContext ctx) {
-      return ctx.runtime.newArray(new IRubyObject[] {mod, unpackerProc, unpackerArg});
+      return ctx.runtime.newArray(new IRubyObject[] {mod, unpackerProc});
     }
 
     public IRubyObject[] toPackerProcTypeIdPair(ThreadContext ctx) {

data/ext/java/org/msgpack/jruby/ExtensionValue.java

@@ -26,7 +26,7 @@ import static org.msgpack.jruby.Types.*;
 @JRubyClass(name="MessagePack::ExtensionValue")
 public class ExtensionValue extends RubyObject {
   private static final long serialVersionUID = 8451274621449322492L;
-  private final Encoding binaryEncoding;
+  private transient final Encoding binaryEncoding;
 
   private RubyFixnum type;
   private RubyString payload;

data/ext/java/org/msgpack/jruby/Factory.java

@@ -26,8 +26,8 @@ import static org.jruby.runtime.Visibility.PRIVATE;
 @JRubyClass(name="MessagePack::Factory")
 public class Factory extends RubyObject {
   private static final long serialVersionUID = 8441284623445322492L;
-  private final Ruby runtime;
-  private ExtensionRegistry extensionRegistry;
+  private transient final Ruby runtime;
+  private transient ExtensionRegistry extensionRegistry;
   private boolean hasSymbolExtType;
   private boolean hasBigIntExtType;
 
@@ -80,39 +80,15 @@ public class Factory extends RubyObject {
     });
   }
 
-  @JRubyMethod(name = "register_type", required = 2, optional = 1)
-  public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args) {
-    Ruby runtime = ctx.runtime;
-    IRubyObject type = args[0];
-    IRubyObject mod = args[1];
-
-    IRubyObject packerArg;
-    IRubyObject unpackerArg;
-
-    RubyHash options = null;
+  @JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
+  public IRubyObject registerTypeInternal(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject opts) {
+    testFrozen("MessagePack::Factory");
 
-    if (isFrozen()) {
-        throw runtime.newRuntimeError("can't modify frozen Factory");
-    }
+    Ruby runtime = ctx.runtime;
+    RubyHash options = (RubyHash) opts;
 
-    if (args.length == 2) {
-      packerArg = runtime.newSymbol("to_msgpack_ext");
-      unpackerArg = runtime.newSymbol("from_msgpack_ext");
-    } else if (args.length == 3) {
-      if (args[args.length - 1] instanceof RubyHash) {
-        options = (RubyHash) args[args.length - 1];
-        packerArg = options.fastARef(runtime.newSymbol("packer"));
-        unpackerArg = options.fastARef(runtime.newSymbol("unpacker"));
-        IRubyObject optimizedSymbolsParsingArg = options.fastARef(runtime.newSymbol("optimized_symbols_parsing"));
-        if (optimizedSymbolsParsingArg != null && optimizedSymbolsParsingArg.isTrue()) {
-          throw runtime.newArgumentError("JRuby implementation does not support the optimized_symbols_parsing option");
-        }
-      } else {
-        throw runtime.newArgumentError(String.format("expected Hash but found %s.", args[args.length - 1].getType().getName()));
-      }
-    } else {
-      throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 2..3)", 2 + args.length));
-    }
+    IRubyObject packerProc = options.fastARef(runtime.newSymbol("packer"));
+    IRubyObject unpackerProc = options.fastARef(runtime.newSymbol("unpacker"));
 
     long typeId = ((RubyFixnum) type).getLongValue();
     if (typeId < -128 || typeId > 127) {
@@ -124,21 +100,6 @@ public class Factory extends RubyObject {
     }
     RubyModule extModule = (RubyModule) mod;
 
-    IRubyObject packerProc = runtime.getNil();
-    IRubyObject unpackerProc = runtime.getNil();
-    if (packerArg != null) {
-      packerProc = packerArg.callMethod(ctx, "to_proc");
-    }
-    if (unpackerArg != null) {
-      if (unpackerArg instanceof RubyString || unpackerArg instanceof RubySymbol) {
-        unpackerProc = extModule.method(unpackerArg.callMethod(ctx, "to_sym"));
-      } else if (unpackerArg instanceof RubyProc || unpackerArg instanceof RubyMethod) {
-        unpackerProc = unpackerArg;
-      } else {
-        unpackerProc = unpackerArg.callMethod(ctx, "method", runtime.newSymbol("call"));
-      }
-    }
-
     boolean recursive = false;
     if (options != null) {
       IRubyObject recursiveExtensionArg = options.fastARef(runtime.newSymbol("recursive"));
@@ -147,9 +108,9 @@ public class Factory extends RubyObject {
       }
     }
 
-    extensionRegistry.put(extModule, (int) typeId, recursive, packerProc, packerArg, unpackerProc, unpackerArg);
+    extensionRegistry.put(extModule, (int) typeId, recursive, packerProc, unpackerProc);
 
-    if (extModule == runtime.getSymbol()) {
+    if (extModule == runtime.getSymbol() && !packerProc.isNil()) {
       hasSymbolExtType = true;
     }
 

data/ext/java/org/msgpack/jruby/Packer.java

@@ -28,12 +28,12 @@ import static org.jruby.runtime.Visibility.PRIVATE;
 @JRubyClass(name="MessagePack::Packer")
 public class Packer extends RubyObject {
   private static final long serialVersionUID = 8451274621499362492L;
-  public ExtensionRegistry registry;
+  public transient ExtensionRegistry registry;
   private Buffer buffer;
-  private Encoder encoder;
+  private transient Encoder encoder;
   private boolean hasSymbolExtType;
   private boolean hasBigintExtType;
-  private Encoding binaryEncoding;
+  private transient Encoding binaryEncoding;
 
   public Packer(Ruby runtime, RubyClass type, ExtensionRegistry registry, boolean hasSymbolExtType, boolean hasBigintExtType) {
     super(runtime, type);
@@ -93,26 +93,11 @@ public class Packer extends RubyObject {
     return registry.toInternalPackerRegistry(ctx);
   }
 
-  @JRubyMethod(name = "register_type", required = 2, optional = 1)
-  public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args, final Block block) {
+  @JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
+  public IRubyObject registerType(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject proc) {
+    testFrozen("MessagePack::Packer");
+
     Ruby runtime = ctx.runtime;
-    IRubyObject type = args[0];
-    IRubyObject mod = args[1];
-
-    IRubyObject arg;
-    IRubyObject proc;
-    if (args.length == 2) {
-      if (! block.isGiven()) {
-        throw runtime.newLocalJumpErrorNoBlock();
-      }
-      proc = block.getProcObject();
-      arg = proc;
-    } else if (args.length == 3) {
-      arg = args[2];
-      proc = arg.callMethod(ctx, "to_proc");
-    } else {
-      throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 2..3)", 2 + args.length));
-    }
 
     long typeId = ((RubyFixnum) type).getLongValue();
     if (typeId < -128 || typeId > 127) {
@@ -124,9 +109,9 @@ public class Packer extends RubyObject {
     }
     RubyModule extModule = (RubyModule) mod;
 
-    registry.put(extModule, (int) typeId, false, proc, arg, null, null);
+    registry.put(extModule, (int) typeId, false, proc, null);
 
-    if (extModule == runtime.getSymbol()) {
+    if (extModule == runtime.getSymbol() && !proc.isNil()) {
       encoder.hasSymbolExtType = true;
     }
 

data/ext/java/org/msgpack/jruby/Unpacker.java

@@ -21,18 +21,17 @@ import org.jruby.runtime.ThreadContext;
 import org.jruby.anno.JRubyClass;
 import org.jruby.anno.JRubyMethod;
 import org.jruby.util.ByteList;
-import org.jruby.ext.stringio.StringIO;
 
 import static org.jruby.runtime.Visibility.PRIVATE;
 
 @JRubyClass(name="MessagePack::Unpacker")
 public class Unpacker extends RubyObject {
   private static final long serialVersionUID = 8451264671199362492L;
-  private final ExtensionRegistry registry;
+  private transient final ExtensionRegistry registry;
 
-  private IRubyObject stream;
-  private IRubyObject data;
-  private Decoder decoder;
+  private transient IRubyObject stream;
+  private transient IRubyObject data;
+  private transient Decoder decoder;
   private final RubyClass underflowErrorClass;
   private boolean symbolizeKeys;
   private boolean freeze;
@@ -127,37 +126,23 @@ public class Unpacker extends RubyObject {
     return registry.toInternalUnpackerRegistry(ctx);
   }
 
-  @JRubyMethod(name = "register_type", required = 1, optional = 2)
-  public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args, final Block block) {
+  @JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
+  public IRubyObject registerTypeInternal(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject proc) {
+    testFrozen("MessagePack::Unpacker");
+
     Ruby runtime = ctx.runtime;
-    IRubyObject type = args[0];
-
-    RubyModule extModule;
-    IRubyObject arg;
-    IRubyObject proc;
-    if (args.length == 1) {
-      if (! block.isGiven()) {
-        throw runtime.newLocalJumpErrorNoBlock();
-      }
-      proc = RubyProc.newProc(runtime, block, block.type);
-      if (proc == null)
-        System.err.println("proc from Block is null");
-      arg = proc;
-      extModule = null;
-    } else if (args.length == 3) {
-      extModule = (RubyModule) args[1];
-      arg = args[2];
-      proc = extModule.method(arg);
-    } else {
-      throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 1 or 3)", 2 + args.length));
-    }
 
     long typeId = ((RubyFixnum) type).getLongValue();
     if (typeId < -128 || typeId > 127) {
       throw runtime.newRangeError(String.format("integer %d too big to convert to `signed char'", typeId));
     }
 
-    registry.put(extModule, (int) typeId, false, null, null, proc, arg);
+    RubyModule extModule = null;
+    if (mod != runtime.getNil()) {
+      extModule = (RubyModule)mod;
+    }
+
+    registry.put(extModule, (int) typeId, false, null, proc);
     return runtime.getNil();
   }
 
@@ -331,9 +316,7 @@ public class Unpacker extends RubyObject {
   @JRubyMethod(name = "stream=", required = 1)
   public IRubyObject setStream(ThreadContext ctx, IRubyObject stream) {
     RubyString str;
-    if (stream instanceof StringIO) {
-      str = stream.callMethod(ctx, "string").asString();
-    } else if (stream instanceof RubyIO) {
+    if (stream instanceof RubyIO) {
       str = stream.callMethod(ctx, "read").asString();
     } else if (stream.respondsTo("read")) {
       str = stream.callMethod(ctx, "read").asString();