msgpack 1.5.6 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c3bd9f87eaddd3212797209da93c4ec3a1c2384cb4a41e0ea9feb48a75523f2
-  data.tar.gz: 16d6223c1c98f50e629c029f077383cbbd7cd194af038255fd28d37aeecbb39a
+  metadata.gz: efdb772bf54b74587a6c99e9513f9c16c78bbef3e5e3c17064a4be79fd5adb7a
+  data.tar.gz: d2f74cb1115947f5337cd730b6283fac61ceeed3a9457597cc90155b3dc93d7e
 SHA512:
-  metadata.gz: 6d4bd9048f852ac4c6e1365b07e6a8df9869cd0d9b2ce2c8bff7c836fe3e513451cf8895b36a719d09beaafdb85101363b705da719e5e9e9c7361ee1de96f2cf
-  data.tar.gz: a15ac8e77af9dc0ccb06ed266efe1ff93a2bae78e2ab5b980ba87af486d08e34676f9203fbc6b646b3c7f14e2fbdf7d4e9e1892fd58d0ca214a39de0dd2ee345
+  metadata.gz: 92da2466eac162f0d6d319496d7af3dbbef0d76b77c3aefb4787213e3b957464603859101ae28ac9620be82feb29278b2ed601adf2fd9d9811b364c258061d4b
+  data.tar.gz: a469586178eb44bbd50abf4cb34684f6a3fd4038063a80d9b3bf7f5cb9e6a2b34c1fed853be669f0bfb356d618efff3a2eaa51b86dd34d5427787a05453b8e11

data/ChangeLog

@@ -1,3 +1,53 @@
+2025-02-06 1.8.0
+
+* Numerous small optimizations.
+* Added `key_cache` option to `Unpacker`.
+
+2024-11-11 1.7.5
+
+* Rerelease 1.7.4 with fixed java package.
+
+2024-11-11 1.7.4
+
+* Fixed a potental memory leak when recursive unpacker raise.
+
+2024-10-03 1.7.3
+
+* Limit initial containers pre-allocation to `SHRT_MAX` (32k) entries.
+
+2023-07-18 1.7.2:
+
+* Fix a potential GC bug when packing data using recursive extensions and buffers containing over 512KkiB of data (See #341).
+* Fix a regression where feeding an empty string to an Unpacker would be considered like the end of the buffer.
+
+2023-05-19 1.7.1:
+
+* Fix JRuby 9.4 compatibility.
+* Fix compilation on older compilers (gcc 4.x).
+* Fix an infinite recursion issue when registering a Symbol type with a `nil` packer.
+
+2023-03-29 1.7.0:
+
+* Fix a possible double-free issue when GC triggers inside `_msgpack_rmem_alloc2`.
+* `Unpacker#feed` now always directly read in provided strings instead of copying content in its buffer.
+* `Unpacker#feed` is now an alias of `Unpacker#feed_reference`.
+* Implement `Factory::Pool#unpacker` and `Factory::Pool#packer` to allow for more precise serialization.
+* Require Ruby 2.5+.
+
+2023-03-03 1.6.1:
+
+* Undefine `#clone` and `#dup` on `MessagePack::Buffer`, `MessagePack::Packer` and `MessagePack::Unpacker`.
+  These methods were never intended, and using them could cause leaks or crashes or worse.
+* Fix a possible GC crash when GC trigger inside `MessagePack::Buffer.new` (#314).
+
+2022-09-30 1.6.0:
+
+* Fix a potential use-after-free bug in Buffer_free when accessing a packer or unpacker buffer. 
+* `old-style-definition` compilation warnings.
+* Restore zero-copy buffer feed when provided a Ruby string. This was accidentally broken in 1.5.4.
+* Provide implementations for `ObjectSpace.memsize`. Message pack objects now properly report their size to Ruby.
+* Fix an endianess bug on Windows platform.
+
 2022-08-23 1.5.6:
 
 * No actual code change, just re-release the `java` version properly.

data/README.md

@@ -8,15 +8,24 @@ and typical short strings only require an extra byte in addition to the strings
 If you ever wished to use JSON for convenience (storing an image with metadata) but could
 not for technical reasons (binary data, size, speed...), MessagePack is a perfect replacement.
 
-    require 'msgpack'
-    msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
-    MessagePack.unpack(msg)   #=> [1,2,3]
+```ruby
+require 'msgpack'
+msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
+MessagePack.unpack(msg)   #=> [1,2,3]
+```
+
+Add msgpack to your Gemfile to install with Bundler:
 
-Use RubyGems to install:
+```ruby
+# Gemfile
+gem 'msgpack'
+```
+
+Or, use RubyGems to install:
 
     gem install msgpack
 
-or build msgpack-ruby and install:
+Or, build msgpack-ruby and install from a checked-out msgpack-ruby repository:
 
     bundle
     rake
@@ -27,11 +36,11 @@ or build msgpack-ruby and install:
 
 * Create REST API returing MessagePack using Rails + [RABL](https://github.com/nesquena/rabl)
 * Store objects efficiently serialized by msgpack on memcached or Redis
-  * In fact Redis supports msgpack in [EVAL-scripts](http://redis.io/commands/eval)
+  * In fact Redis supports msgpack in [EVAL-scripts](https://redis.io/docs/latest/commands/eval/)
 * Upload data in efficient format from mobile devices such as smartphones
   * MessagePack works on iPhone/iPad and Android. See also [Objective-C](https://github.com/msgpack/msgpack-objectivec) and [Java](https://github.com/msgpack/msgpack-java) implementations
 * Design a portable protocol to communicate with embedded devices
-  * Check also [Fluentd](http://fluentd.org/) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
+  * Check also [Fluentd](https://www.fluentd.org) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
 * Exchange objects between software components written in different languages
   * You'll need a flexible but efficient format so that components exchange objects while keeping compatibility
 
@@ -128,9 +137,9 @@ being serialized altogether by throwing an exception:
 
 ```ruby
 class Symbol
-    def to_msgpack_ext
-        raise "Serialization of symbols prohibited"
-    end
+  def to_msgpack_ext
+    raise "Serialization of symbols prohibited"
+  end
 end
 
 MessagePack::DefaultFactory.register_type(0x00, Symbol)
@@ -211,6 +220,33 @@ factory.register_type(
 factory.load(factory.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
 ```
 
+## Pooling
+
+Creating `Packer` and `Unpacker` objects is expensive. For best performance it is preferable to re-use these objects.
+
+`MessagePack::Factory#pool` makes that easier:
+
+```ruby
+factory = MessagePack::Factory.new
+factory.register_type(
+  0x01,
+  Point,
+  packer: ->(point, packer) {
+    packer.write(point.x)
+    packer.write(point.y)
+  },
+  unpacker: ->(unpacker) {
+    x = unpacker.read
+    y = unpacker.read
+    Point.new(x, y)
+  },
+  recursive: true,
+)
+pool = factory.pool(5) # The pool size should match the number of threads expected to use the factory concurrently.
+
+pool.load(pool.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
+```
+
 ## Buffer API
 
 MessagePack for Ruby provides a buffer API so that you can read or write data by hand, not via Packer or Unpacker API.
@@ -249,8 +285,8 @@ If this directory has Gemfile.lock (generated with MRI), remove it beforehand.
 
 ## Updating documents
 
-Online documents (http://ruby.msgpack.org) is generated from gh-pages branch.
-Following commands update documents in gh-pages branch:
+Online documentation (https://ruby.msgpack.org) is generated from the gh-pages branch.
+To update documents in gh-pages branch:
 
     bundle exec rake doc
     git checkout gh-pages

data/ext/java/org/msgpack/jruby/Buffer.java

@@ -22,10 +22,10 @@ import org.jcodings.Encoding;
 @JRubyClass(name="MessagePack::Buffer")
 public class Buffer extends RubyObject {
   private static final long serialVersionUID = 8441244627425629412L;
-  private IRubyObject io;
-  private ByteBuffer buffer;
+  private transient IRubyObject io;
+  private transient ByteBuffer buffer;
   private boolean writeMode;
-  private Encoding binaryEncoding;
+  private transient Encoding binaryEncoding;
 
   private static final int CACHE_LINE_SIZE = 64;
   private static final int ARRAY_HEADER_SIZE = 24;

data/ext/java/org/msgpack/jruby/ExtensionRegistry.java

@@ -18,22 +18,17 @@ public class ExtensionRegistry {
   private final ExtensionEntry[] extensionsByTypeId;
 
   public ExtensionRegistry() {
-    this(new HashMap<RubyModule, ExtensionEntry>());
+    this(new HashMap<RubyModule, ExtensionEntry>(), new ExtensionEntry[256]);
   }
 
-  private ExtensionRegistry(Map<RubyModule, ExtensionEntry> extensionsByModule) {
+  private ExtensionRegistry(Map<RubyModule, ExtensionEntry> extensionsByModule, ExtensionEntry[] extensionsByTypeId) {
     this.extensionsByModule = new HashMap<RubyModule, ExtensionEntry>(extensionsByModule);
     this.extensionsByAncestor = new HashMap<RubyModule, ExtensionEntry>();
-    this.extensionsByTypeId = new ExtensionEntry[256];
-    for (ExtensionEntry entry : extensionsByModule.values()) {
-      if (entry.hasUnpacker()) {
-        extensionsByTypeId[entry.getTypeId() + 128] = entry;
-      }
-    }
+    this.extensionsByTypeId = extensionsByTypeId.clone();
   }
 
   public ExtensionRegistry dup() {
-    return new ExtensionRegistry(extensionsByModule);
+    return new ExtensionRegistry(extensionsByModule, extensionsByTypeId);
   }
 
   public IRubyObject toInternalPackerRegistry(ThreadContext ctx) {
@@ -59,8 +54,8 @@ public class ExtensionRegistry {
     return hash;
   }
 
-  public void put(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject packerArg, IRubyObject unpackerProc, IRubyObject unpackerArg) {
-    ExtensionEntry entry = new ExtensionEntry(mod, typeId, recursive, packerProc, packerArg, unpackerProc, unpackerArg);
+  public void put(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject unpackerProc) {
+    ExtensionEntry entry = new ExtensionEntry(mod, typeId, recursive, packerProc, unpackerProc);
     extensionsByModule.put(mod, entry);
     extensionsByTypeId[typeId + 128] = entry;
     extensionsByAncestor.clear();
@@ -119,18 +114,14 @@ public class ExtensionRegistry {
     private final int typeId;
     private final boolean recursive;
     private final IRubyObject packerProc;
-    private final IRubyObject packerArg;
     private final IRubyObject unpackerProc;
-    private final IRubyObject unpackerArg;
 
-    public ExtensionEntry(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject packerArg, IRubyObject unpackerProc, IRubyObject unpackerArg) {
+    public ExtensionEntry(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject unpackerProc) {
       this.mod = mod;
       this.typeId = typeId;
       this.recursive = recursive;
       this.packerProc = packerProc;
-      this.packerArg = packerArg;
       this.unpackerProc = unpackerProc;
-      this.unpackerArg = unpackerArg;
     }
 
     public RubyModule getExtensionModule() {
@@ -146,11 +137,11 @@ public class ExtensionRegistry {
     }
 
     public boolean hasPacker() {
-      return packerProc != null;
+      return packerProc != null && !packerProc.isNil();
     }
 
     public boolean hasUnpacker() {
-      return unpackerProc != null;
+      return unpackerProc != null && !unpackerProc.isNil();
     }
 
     public IRubyObject getPackerProc() {
@@ -162,11 +153,11 @@ public class ExtensionRegistry {
     }
 
     public RubyArray<?> toPackerTuple(ThreadContext ctx) {
-      return ctx.runtime.newArray(new IRubyObject[] {ctx.runtime.newFixnum(typeId), packerProc, packerArg});
+      return ctx.runtime.newArray(new IRubyObject[] {ctx.runtime.newFixnum(typeId), packerProc});
     }
 
     public RubyArray<?> toUnpackerTuple(ThreadContext ctx) {
-      return ctx.runtime.newArray(new IRubyObject[] {mod, unpackerProc, unpackerArg});
+      return ctx.runtime.newArray(new IRubyObject[] {mod, unpackerProc});
     }
 
     public IRubyObject[] toPackerProcTypeIdPair(ThreadContext ctx) {

data/ext/java/org/msgpack/jruby/ExtensionValue.java

@@ -26,7 +26,7 @@ import static org.msgpack.jruby.Types.*;
 @JRubyClass(name="MessagePack::ExtensionValue")
 public class ExtensionValue extends RubyObject {
   private static final long serialVersionUID = 8451274621449322492L;
-  private final Encoding binaryEncoding;
+  private transient final Encoding binaryEncoding;
 
   private RubyFixnum type;
   private RubyString payload;

data/ext/java/org/msgpack/jruby/Factory.java

@@ -26,8 +26,8 @@ import static org.jruby.runtime.Visibility.PRIVATE;
 @JRubyClass(name="MessagePack::Factory")
 public class Factory extends RubyObject {
   private static final long serialVersionUID = 8441284623445322492L;
-  private final Ruby runtime;
-  private ExtensionRegistry extensionRegistry;
+  private transient final Ruby runtime;
+  private transient ExtensionRegistry extensionRegistry;
   private boolean hasSymbolExtType;
   private boolean hasBigIntExtType;
 
@@ -80,39 +80,15 @@ public class Factory extends RubyObject {
     });
   }
 
-  @JRubyMethod(name = "register_type", required = 2, optional = 1)
-  public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args) {
-    Ruby runtime = ctx.runtime;
-    IRubyObject type = args[0];
-    IRubyObject mod = args[1];
-
-    IRubyObject packerArg;
-    IRubyObject unpackerArg;
-
-    RubyHash options = null;
+  @JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
+  public IRubyObject registerTypeInternal(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject opts) {
+    testFrozen("MessagePack::Factory");
 
-    if (isFrozen()) {
-        throw runtime.newRuntimeError("can't modify frozen Factory");
-    }
+    Ruby runtime = ctx.runtime;
+    RubyHash options = (RubyHash) opts;
 
-    if (args.length == 2) {
-      packerArg = runtime.newSymbol("to_msgpack_ext");
-      unpackerArg = runtime.newSymbol("from_msgpack_ext");
-    } else if (args.length == 3) {
-      if (args[args.length - 1] instanceof RubyHash) {
-        options = (RubyHash) args[args.length - 1];
-        packerArg = options.fastARef(runtime.newSymbol("packer"));
-        unpackerArg = options.fastARef(runtime.newSymbol("unpacker"));
-        IRubyObject optimizedSymbolsParsingArg = options.fastARef(runtime.newSymbol("optimized_symbols_parsing"));
-        if (optimizedSymbolsParsingArg != null && optimizedSymbolsParsingArg.isTrue()) {
-          throw runtime.newArgumentError("JRuby implementation does not support the optimized_symbols_parsing option");
-        }
-      } else {
-        throw runtime.newArgumentError(String.format("expected Hash but found %s.", args[args.length - 1].getType().getName()));
-      }
-    } else {
-      throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 2..3)", 2 + args.length));
-    }
+    IRubyObject packerProc = options.fastARef(runtime.newSymbol("packer"));
+    IRubyObject unpackerProc = options.fastARef(runtime.newSymbol("unpacker"));
 
     long typeId = ((RubyFixnum) type).getLongValue();
     if (typeId < -128 || typeId > 127) {
@@ -124,21 +100,6 @@ public class Factory extends RubyObject {
     }
     RubyModule extModule = (RubyModule) mod;
 
-    IRubyObject packerProc = runtime.getNil();
-    IRubyObject unpackerProc = runtime.getNil();
-    if (packerArg != null) {
-      packerProc = packerArg.callMethod(ctx, "to_proc");
-    }
-    if (unpackerArg != null) {
-      if (unpackerArg instanceof RubyString || unpackerArg instanceof RubySymbol) {
-        unpackerProc = extModule.method(unpackerArg.callMethod(ctx, "to_sym"));
-      } else if (unpackerArg instanceof RubyProc || unpackerArg instanceof RubyMethod) {
-        unpackerProc = unpackerArg;
-      } else {
-        unpackerProc = unpackerArg.callMethod(ctx, "method", runtime.newSymbol("call"));
-      }
-    }
-
     boolean recursive = false;
     if (options != null) {
       IRubyObject recursiveExtensionArg = options.fastARef(runtime.newSymbol("recursive"));
@@ -147,9 +108,9 @@ public class Factory extends RubyObject {
       }
     }
 
-    extensionRegistry.put(extModule, (int) typeId, recursive, packerProc, packerArg, unpackerProc, unpackerArg);
+    extensionRegistry.put(extModule, (int) typeId, recursive, packerProc, unpackerProc);
 
-    if (extModule == runtime.getSymbol()) {
+    if (extModule == runtime.getSymbol() && !packerProc.isNil()) {
       hasSymbolExtType = true;
     }
 

data/ext/java/org/msgpack/jruby/Packer.java

@@ -28,12 +28,12 @@ import static org.jruby.runtime.Visibility.PRIVATE;
 @JRubyClass(name="MessagePack::Packer")
 public class Packer extends RubyObject {
   private static final long serialVersionUID = 8451274621499362492L;
-  public ExtensionRegistry registry;
+  public transient ExtensionRegistry registry;
   private Buffer buffer;
-  private Encoder encoder;
+  private transient Encoder encoder;
   private boolean hasSymbolExtType;
   private boolean hasBigintExtType;
-  private Encoding binaryEncoding;
+  private transient Encoding binaryEncoding;
 
   public Packer(Ruby runtime, RubyClass type, ExtensionRegistry registry, boolean hasSymbolExtType, boolean hasBigintExtType) {
     super(runtime, type);
@@ -93,26 +93,11 @@ public class Packer extends RubyObject {
     return registry.toInternalPackerRegistry(ctx);
   }
 
-  @JRubyMethod(name = "register_type", required = 2, optional = 1)
-  public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args, final Block block) {
+  @JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
+  public IRubyObject registerType(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject proc) {
+    testFrozen("MessagePack::Packer");
+
     Ruby runtime = ctx.runtime;
-    IRubyObject type = args[0];
-    IRubyObject mod = args[1];
-
-    IRubyObject arg;
-    IRubyObject proc;
-    if (args.length == 2) {
-      if (! block.isGiven()) {
-        throw runtime.newLocalJumpErrorNoBlock();
-      }
-      proc = block.getProcObject();
-      arg = proc;
-    } else if (args.length == 3) {
-      arg = args[2];
-      proc = arg.callMethod(ctx, "to_proc");
-    } else {
-      throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 2..3)", 2 + args.length));
-    }
 
     long typeId = ((RubyFixnum) type).getLongValue();
     if (typeId < -128 || typeId > 127) {
@@ -124,9 +109,9 @@ public class Packer extends RubyObject {
     }
     RubyModule extModule = (RubyModule) mod;
 
-    registry.put(extModule, (int) typeId, false, proc, arg, null, null);
+    registry.put(extModule, (int) typeId, false, proc, null);
 
-    if (extModule == runtime.getSymbol()) {
+    if (extModule == runtime.getSymbol() && !proc.isNil()) {
       encoder.hasSymbolExtType = true;
     }
 

data/ext/java/org/msgpack/jruby/Unpacker.java

@@ -21,18 +21,17 @@ import org.jruby.runtime.ThreadContext;
 import org.jruby.anno.JRubyClass;
 import org.jruby.anno.JRubyMethod;
 import org.jruby.util.ByteList;
-import org.jruby.ext.stringio.StringIO;
 
 import static org.jruby.runtime.Visibility.PRIVATE;
 
 @JRubyClass(name="MessagePack::Unpacker")
 public class Unpacker extends RubyObject {
   private static final long serialVersionUID = 8451264671199362492L;
-  private final ExtensionRegistry registry;
+  private transient final ExtensionRegistry registry;
 
-  private IRubyObject stream;
-  private IRubyObject data;
-  private Decoder decoder;
+  private transient IRubyObject stream;
+  private transient IRubyObject data;
+  private transient Decoder decoder;
   private final RubyClass underflowErrorClass;
   private boolean symbolizeKeys;
   private boolean freeze;
@@ -127,37 +126,23 @@ public class Unpacker extends RubyObject {
     return registry.toInternalUnpackerRegistry(ctx);
   }
 
-  @JRubyMethod(name = "register_type", required = 1, optional = 2)
-  public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args, final Block block) {
+  @JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
+  public IRubyObject registerTypeInternal(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject proc) {
+    testFrozen("MessagePack::Unpacker");
+
     Ruby runtime = ctx.runtime;
-    IRubyObject type = args[0];
-
-    RubyModule extModule;
-    IRubyObject arg;
-    IRubyObject proc;
-    if (args.length == 1) {
-      if (! block.isGiven()) {
-        throw runtime.newLocalJumpErrorNoBlock();
-      }
-      proc = RubyProc.newProc(runtime, block, block.type);
-      if (proc == null)
-        System.err.println("proc from Block is null");
-      arg = proc;
-      extModule = null;
-    } else if (args.length == 3) {
-      extModule = (RubyModule) args[1];
-      arg = args[2];
-      proc = extModule.method(arg);
-    } else {
-      throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 1 or 3)", 2 + args.length));
-    }
 
     long typeId = ((RubyFixnum) type).getLongValue();
     if (typeId < -128 || typeId > 127) {
       throw runtime.newRangeError(String.format("integer %d too big to convert to `signed char'", typeId));
     }
 
-    registry.put(extModule, (int) typeId, false, null, null, proc, arg);
+    RubyModule extModule = null;
+    if (mod != runtime.getNil()) {
+      extModule = (RubyModule)mod;
+    }
+
+    registry.put(extModule, (int) typeId, false, null, proc);
     return runtime.getNil();
   }
 
@@ -331,9 +316,7 @@ public class Unpacker extends RubyObject {
   @JRubyMethod(name = "stream=", required = 1)
   public IRubyObject setStream(ThreadContext ctx, IRubyObject stream) {
     RubyString str;
-    if (stream instanceof StringIO) {
-      str = stream.callMethod(ctx, "string").asString();
-    } else if (stream instanceof RubyIO) {
+    if (stream instanceof RubyIO) {
       str = stream.callMethod(ctx, "read").asString();
     } else if (stream.respondsTo("read")) {
       str = stream.callMethod(ctx, "read").asString();