msgpack 1.4.2 → 1.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 451ed743fe37bd3726f897f4daae9e7fd9b2cb065a34cf438ca786350060e0ec
-  data.tar.gz: 87335c4b1d35f9ac88af648963b1fe60caa72059ec7e6c1b6b507027da9118d8
+  metadata.gz: fe6adcd787be2f8c3e9459d46c3baa4ad81c5b08bdcaeb9bfec9a80c9a4540c6
+  data.tar.gz: 76d6be5a37a2b6e225ad995e6b0f670d0489839a98b678c9332837bd082e7d7e
 SHA512:
-  metadata.gz: eb519049e9d8e4b36682e4b2c2625f10172745ce74ad1cfc8e6d93468b219071ed579bcfd36b7f0e1210eeb177197b435da2a028f46d2c71c73a3bcf2b53dcf8
-  data.tar.gz: e8c4765ac836226f5a2ce68a62194f112fc77bcdd9744d39040ff2db5a2c27d876348fb6002110c00b122bccdff2d539223d82ff73ea0bcb4cf1a3d9fb62defe
+  metadata.gz: 36b0e748e7b54baad2ecef8c1c071a82cf3e6bd26540fd433a3c71f1e77afe341188594a26bfed817f84af10ef4f613ded14fc0a8448a77dc4978683cf7c35d1
+  data.tar.gz: 45ce3770a12b3580945ba3dd42d40ab8ba24a167f675f71d57bb42af6cf4ab204d7cc7960925ea78556988f523886fd92b431789a174e0529143ab0908d52418

data/ChangeLog

@@ -1,3 +1,92 @@
+2024-10-03 1.7.3
+
+* Limit initial containers pre-allocation to `SHRT_MAX` (32k) entries.
+
+2023-07-18 1.7.2:
+
+* Fix a potential GC bug when packing data using recursive extensions and buffers containing over 512KkiB of data (See #341).
+* Fix a regression where feeding an empty string to an Unpacker would be considered like the end of the buffer.
+
+2023-05-19 1.7.1:
+
+* Fix JRuby 9.4 compatibility.
+* Fix compilation on older compilers (gcc 4.x).
+* Fix an infinite recursion issue when registering a Symbol type with a `nil` packer.
+
+2023-03-29 1.7.0:
+
+* Fix a possible double-free issue when GC triggers inside `_msgpack_rmem_alloc2`.
+* `Unpacker#feed` now always directly read in provided strings instead of copying content in its buffer.
+* `Unpacker#feed` is now an alias of `Unpacker#feed_reference`.
+* Implement `Factory::Pool#unpacker` and `Factory::Pool#packer` to allow for more precise serialization.
+* Require Ruby 2.5+.
+
+2023-03-03 1.6.1:
+
+* Undefine `#clone` and `#dup` on `MessagePack::Buffer`, `MessagePack::Packer` and `MessagePack::Unpacker`.
+  These methods were never intended, and using them could cause leaks or crashes or worse.
+* Fix a possible GC crash when GC trigger inside `MessagePack::Buffer.new` (#314).
+
+2022-09-30 1.6.0:
+
+* Fix a potential use-after-free bug in Buffer_free when accessing a packer or unpacker buffer. 
+* `old-style-definition` compilation warnings.
+* Restore zero-copy buffer feed when provided a Ruby string. This was accidentally broken in 1.5.4.
+* Provide implementations for `ObjectSpace.memsize`. Message pack objects now properly report their size to Ruby.
+* Fix an endianess bug on Windows platform.
+
+2022-08-23 1.5.6:
+
+* No actual code change, just re-release the `java` version properly.
+
+2022-08-22 1.5.5:
+
+* Fix a segfault when GC triggers inside a recursive extension.
+
+2022-07-25 1.5.4:
+
+* Fix a segfault when deserializing empty symbol (`:""`).
+* Improve compilation flags to not strip debug symbols.
+
+2022-05-30 version 1.5.3:
+
+* Fix deduplication of empty strings when using the `freeze: true` option.
+* Use `rb_hash_new_capa` when available (Ruby 3.2) for improved performance when parsing large hashes.
+
+2022-05-27 version 1.5.2:
+
+* Fix bug about unpacking ext type objects with the recursive option
+
+2022-04-07 version 1.5.1:
+
+* Fix bug about packing/unpacking ext type objects with the recursive option
+
+2022-04-06 version 1.5.0:
+
+* Add recursive option on Factory#register_type to operate Packer/Unpacker manually
+* Add oversized_integer_extension option on Factory#register_type to pack/unpack bigint using ext types
+* Add Factory#pool method and Factory::Pool class to provide pooled Packer and Unpacker instances
+
+2022-02-15 version 1.4.5:
+
+* Fix to create UTF-8 Symbol keys when symbolize_keys: true
+* Fix to assume Symbols as US-ASCII or UTF-8
+* Optimize Packer/Unpacker initialization
+* Optimize extension class lookup
+* Rename Packer#clear as Packer#reset (#clear is still available as an alias)
+
+2022-01-22 version 1.4.4:
+
+* Specify the build option --platform=8 for older Java platforms
+
+2022-01-20 version 1.4.3:
+
+* Optimize serialization/deserialization of Symbols
+* Support registering ext types for objects of subclasses of primitive types (like Hash)
+* Add optimized_symbols_parsing option to Factory#register_type on MRI implementation
+* Optimize to deduplicate Hash keys on JRuby
+* Support JRuby 9.3 (and drop 9.1)
+
 2021-02-01 version 1.4.2:
 
 * Add the required Ruby version (>= 2.4) to avoid compilation errors on older Ruby runtimes

data/README.md

@@ -8,15 +8,24 @@ and typical short strings only require an extra byte in addition to the strings
 If you ever wished to use JSON for convenience (storing an image with metadata) but could
 not for technical reasons (binary data, size, speed...), MessagePack is a perfect replacement.
 
-    require 'msgpack'
-    msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
-    MessagePack.unpack(msg)   #=> [1,2,3]
+```ruby
+require 'msgpack'
+msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
+MessagePack.unpack(msg)   #=> [1,2,3]
+```
 
-Use RubyGems to install:
+Add msgpack to your Gemfile to install with Bundler:
+
+```ruby
+# Gemfile
+gem 'msgpack'
+```
+
+Or, use RubyGems to install:
 
     gem install msgpack
 
-or build msgpack-ruby and install:
+Or, build msgpack-ruby and install from a checked-out msgpack-ruby repository:
 
     bundle
     rake
@@ -27,11 +36,11 @@ or build msgpack-ruby and install:
 
 * Create REST API returing MessagePack using Rails + [RABL](https://github.com/nesquena/rabl)
 * Store objects efficiently serialized by msgpack on memcached or Redis
-  * In fact Redis supports msgpack in [EVAL-scripts](http://redis.io/commands/eval)
+  * In fact Redis supports msgpack in [EVAL-scripts](https://redis.io/docs/latest/commands/eval/)
 * Upload data in efficient format from mobile devices such as smartphones
   * MessagePack works on iPhone/iPad and Android. See also [Objective-C](https://github.com/msgpack/msgpack-objectivec) and [Java](https://github.com/msgpack/msgpack-java) implementations
 * Design a portable protocol to communicate with embedded devices
-  * Check also [Fluentd](http://fluentd.org/) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
+  * Check also [Fluentd](https://www.fluentd.org) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
 * Exchange objects between software components written in different languages
   * You'll need a flexible but efficient format so that components exchange objects while keeping compatibility
 
@@ -40,7 +49,7 @@ or build msgpack-ruby and install:
 MessagePack for Ruby should run on x86, ARM, PowerPC, SPARC and other CPU architectures.
 
 And it works with MRI (CRuby) and Rubinius.
-Patches to improve portability is highly welcomed.
+Patches to improve portability are highly welcomed.
 
 
 ## Serializing objects
@@ -51,6 +60,7 @@ Use `MessagePack.pack` or `to_msgpack`:
 require 'msgpack'
 msg = MessagePack.pack(obj)  # or
 msg = obj.to_msgpack
+File.binwrite('mydata.msgpack', msg)
 ```
 
 ### Streaming serialization
@@ -71,6 +81,7 @@ Use `MessagePack.unpack`:
 
 ```ruby
 require 'msgpack'
+msg = File.binread('mydata.msgpack')
 obj = MessagePack.unpack(msg)
 ```
 
@@ -126,9 +137,9 @@ being serialized altogether by throwing an exception:
 
 ```ruby
 class Symbol
-    def to_msgpack_ext
-        raise "Serialization of symbols prohibited"
-    end
+  def to_msgpack_ext
+    raise "Serialization of symbols prohibited"
+  end
 end
 
 MessagePack::DefaultFactory.register_type(0x00, Symbol)
@@ -187,6 +198,55 @@ MessagePack::DefaultFactory.register_type(0x03, MyClass3)
 MessagePack.unpack(data_with_ext_typeid_03) #=> MyClass3 instance
 ```
 
+Alternatively, extension types can call the packer or unpacker recursively to generate the extension data:
+
+```ruby
+Point = Struct.new(:x, :y)
+factory = MessagePack::Factory.new
+factory.register_type(
+  0x01,
+  Point,
+  packer: ->(point, packer) {
+    packer.write(point.x)
+    packer.write(point.y)
+  },
+  unpacker: ->(unpacker) {
+    x = unpacker.read
+    y = unpacker.read
+    Point.new(x, y)
+  },
+  recursive: true,
+)
+factory.load(factory.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
+```
+
+## Pooling
+
+Creating `Packer` and `Unpacker` objects is expensive. For best performance it is preferable to re-use these objects.
+
+`MessagePack::Factory#pool` makes that easier:
+
+```ruby
+factory = MessagePack::Factory.new
+factory.register_type(
+  0x01,
+  Point,
+  packer: ->(point, packer) {
+    packer.write(point.x)
+    packer.write(point.y)
+  },
+  unpacker: ->(unpacker) {
+    x = unpacker.read
+    y = unpacker.read
+    Point.new(x, y)
+  },
+  recursive: true,
+)
+pool = factory.pool(5) # The pool size should match the number of threads expected to use the factory concurrently.
+
+pool.load(pool.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
+```
+
 ## Buffer API
 
 MessagePack for Ruby provides a buffer API so that you can read or write data by hand, not via Packer or Unpacker API.
@@ -225,8 +285,8 @@ If this directory has Gemfile.lock (generated with MRI), remove it beforehand.
 
 ## Updating documents
 
-Online documents (http://ruby.msgpack.org) is generated from gh-pages branch.
-Following commands update documents in gh-pages branch:
+Online documentation (https://ruby.msgpack.org) is generated from the gh-pages branch.
+To update documents in gh-pages branch:
 
     bundle exec rake doc
     git checkout gh-pages

data/ext/java/org/msgpack/jruby/Buffer.java

@@ -21,10 +21,11 @@ import org.jcodings.Encoding;
 
 @JRubyClass(name="MessagePack::Buffer")
 public class Buffer extends RubyObject {
-  private IRubyObject io;
-  private ByteBuffer buffer;
+  private static final long serialVersionUID = 8441244627425629412L;
+  private transient IRubyObject io;
+  private transient ByteBuffer buffer;
   private boolean writeMode;
-  private Encoding binaryEncoding;
+  private transient Encoding binaryEncoding;
 
   private static final int CACHE_LINE_SIZE = 64;
   private static final int ARRAY_HEADER_SIZE = 24;
@@ -49,7 +50,7 @@ public class Buffer extends RubyObject {
     }
     this.buffer = ByteBuffer.allocate(CACHE_LINE_SIZE - ARRAY_HEADER_SIZE);
     this.writeMode = true;
-    this.binaryEncoding = ctx.getRuntime().getEncodingService().getAscii8bitEncoding();
+    this.binaryEncoding = ctx.runtime.getEncodingService().getAscii8bitEncoding();
     return this;
   }
 
@@ -87,17 +88,17 @@ public class Buffer extends RubyObject {
       writeMode = true;
     }
     buffer.clear();
-    return ctx.getRuntime().getNil();
+    return ctx.runtime.getNil();
   }
 
   @JRubyMethod(name = "size")
   public IRubyObject size(ThreadContext ctx) {
-    return ctx.getRuntime().newFixnum(rawSize());
+    return ctx.runtime.newFixnum(rawSize());
   }
 
   @JRubyMethod(name = "empty?")
   public IRubyObject isEmpty(ThreadContext ctx) {
-    return rawSize() == 0 ? ctx.getRuntime().getTrue() : ctx.getRuntime().getFalse();
+    return rawSize() == 0 ? ctx.runtime.getTrue() : ctx.runtime.getFalse();
   }
 
   private IRubyObject bufferWrite(ThreadContext ctx, IRubyObject str) {
@@ -105,7 +106,7 @@ public class Buffer extends RubyObject {
     int length = bytes.length();
     ensureRemainingCapacity(length);
     buffer.put(bytes.unsafeBytes(), bytes.begin(), length);
-    return ctx.getRuntime().newFixnum(length);
+    return ctx.runtime.newFixnum(length);
 
   }
 
@@ -131,19 +132,19 @@ public class Buffer extends RubyObject {
       length = (int) args[0].convertToInteger().getLongValue();
     }
     if (raiseOnUnderflow && rawSize() < length) {
-      throw ctx.getRuntime().newEOFError();
+      throw ctx.runtime.newEOFError();
     }
     int readLength = Math.min(length, rawSize());
     if (readLength == 0 && length > 0) {
-      return ctx.getRuntime().getNil();
+      return ctx.runtime.getNil();
     } else if (readLength == 0) {
-      return ctx.getRuntime().newString();
+      return ctx.runtime.newString();
     } else {
       ensureReadMode();
       byte[] bytes = new byte[readLength];
       buffer.get(bytes);
       ByteList byteList = new ByteList(bytes, binaryEncoding);
-      return ctx.getRuntime().newString(byteList);
+      return ctx.runtime.newString(byteList);
     }
   }
 
@@ -161,12 +162,12 @@ public class Buffer extends RubyObject {
     feed(ctx);
     int length = (int) _length.convertToInteger().getLongValue();
     if (raiseOnUnderflow && rawSize() < length) {
-      throw ctx.getRuntime().newEOFError();
+      throw ctx.runtime.newEOFError();
     }
     ensureReadMode();
     int skipLength = Math.min(length, rawSize());
     buffer.position(buffer.position() + skipLength);
-    return ctx.getRuntime().newFixnum(skipLength);
+    return ctx.runtime.newFixnum(skipLength);
   }
 
   @JRubyMethod(name = "skip")
@@ -188,23 +189,23 @@ public class Buffer extends RubyObject {
     ensureReadMode();
     int length = buffer.limit() - buffer.position();
     ByteList str = new ByteList(buffer.array(), buffer.position(), length, binaryEncoding, true);
-    return ctx.getRuntime().newString(str);
+    return ctx.runtime.newString(str);
   }
 
   @JRubyMethod(name = "to_a")
   public IRubyObject toA(ThreadContext ctx) {
-    return ctx.getRuntime().newArray(toS(ctx));
+    return ctx.runtime.newArray(toS(ctx));
   }
 
   @JRubyMethod(name = "io")
   public IRubyObject getIo(ThreadContext ctx) {
-    return io == null ? ctx.getRuntime().getNil() : io;
+    return io == null ? ctx.runtime.getNil() : io;
   }
 
   @JRubyMethod(name = "flush")
   public IRubyObject flush(ThreadContext ctx) {
     if (io == null) {
-      return ctx.getRuntime().getNil();
+      return ctx.runtime.getNil();
     } else {
       return io.callMethod(ctx, "flush");
     }
@@ -213,7 +214,7 @@ public class Buffer extends RubyObject {
   @JRubyMethod(name = "close")
   public IRubyObject close(ThreadContext ctx) {
     if (io == null) {
-      return ctx.getRuntime().getNil();
+      return ctx.runtime.getNil();
     } else {
       return io.callMethod(ctx, "close");
     }
@@ -223,4 +224,10 @@ public class Buffer extends RubyObject {
   public IRubyObject writeTo(ThreadContext ctx, IRubyObject io) {
     return io.callMethod(ctx, "write", readCommon(ctx, null, false));
   }
+
+  public ByteList getBytes() {
+    byte[] bytes = new byte[rawSize()];
+    buffer.get(bytes);
+    return new ByteList(bytes, binaryEncoding);
+  }
 }

data/ext/java/org/msgpack/jruby/Decoder.java

@@ -14,6 +14,7 @@ import org.jruby.RubyBignum;
 import org.jruby.RubyString;
 import org.jruby.RubyArray;
 import org.jruby.RubyHash;
+import org.jruby.RubyInteger;
 import org.jruby.exceptions.RaiseException;
 import org.jruby.runtime.builtin.IRubyObject;
 import org.jruby.util.ByteList;
@@ -35,7 +36,7 @@ public class Decoder implements Iterator<IRubyObject> {
   private final RubyClass unexpectedTypeErrorClass;
   private final RubyClass unknownExtTypeErrorClass;
 
-  private ExtensionRegistry registry;
+  private Unpacker unpacker;
   private ByteBuffer buffer;
   private boolean symbolizeKeys;
   private boolean freeze;
@@ -45,29 +46,29 @@ public class Decoder implements Iterator<IRubyObject> {
     this(runtime, null, new byte[] {}, 0, 0, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry) {
-    this(runtime, registry, new byte[] {}, 0, 0, false, false, false);
+  public Decoder(Ruby runtime, Unpacker unpacker) {
+    this(runtime, unpacker, new byte[] {}, 0, 0, false, false, false);
   }
 
   public Decoder(Ruby runtime, byte[] bytes) {
     this(runtime, null, bytes, 0, bytes.length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes) {
-    this(runtime, registry, bytes, 0, bytes.length, false, false, false);
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes) {
+    this(runtime, unpacker, bytes, 0, bytes.length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
-    this(runtime, registry, bytes, 0, bytes.length, symbolizeKeys, freeze, allowUnknownExt);
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
+    this(runtime, unpacker, bytes, 0, bytes.length, symbolizeKeys, freeze, allowUnknownExt);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length) {
-    this(runtime, registry, bytes, offset, length, false, false, false);
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes, int offset, int length) {
+    this(runtime, unpacker, bytes, offset, length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes, int offset, int length, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
     this.runtime = runtime;
-    this.registry = registry;
+    this.unpacker = unpacker;
     this.symbolizeKeys = symbolizeKeys;
     this.freeze = freeze;
     this.allowUnknownExt = allowUnknownExt;
@@ -122,7 +123,6 @@ public class Decoder implements Iterator<IRubyObject> {
     ByteList byteList = new ByteList(bytes, encoding);
     RubyString string = runtime.newString(byteList);
     if (this.freeze) {
-      string.setFrozen(true);
       string = runtime.freezeAndDedupString(string);
     }
     return string;
@@ -140,9 +140,14 @@ public class Decoder implements Iterator<IRubyObject> {
     RubyHash hash = RubyHash.newHash(runtime);
     for (int i = 0; i < size; i++) {
       IRubyObject key = next();
-      if (this.symbolizeKeys && key instanceof RubyString) {
+      if (key instanceof RubyString) {
+        if (this.symbolizeKeys) {
           key = ((RubyString) key).intern();
+        } else {
+          key = runtime.freezeAndDedupString((RubyString) key);
+        }
       }
+
       hash.fastASet(key, next());
     }
     return hash;
@@ -150,18 +155,21 @@ public class Decoder implements Iterator<IRubyObject> {
 
   private IRubyObject consumeExtension(int size) {
     int type = buffer.get();
-    byte[] payload = readBytes(size);
-
-    if (registry != null) {
-      IRubyObject proc = registry.lookupUnpackerByTypeId(type);
-      if (proc != null) {
-        ByteList byteList = new ByteList(payload, runtime.getEncodingService().getAscii8bitEncoding());
-        return proc.callMethod(runtime.getCurrentContext(), "call", runtime.newString(byteList));
+    if (unpacker != null) {
+      ExtensionRegistry.ExtensionEntry entry = unpacker.lookupExtensionByTypeId(type);
+      if (entry != null) {
+        IRubyObject proc = entry.getUnpackerProc();
+        if (entry.isRecursive()) {
+          return proc.callMethod(runtime.getCurrentContext(), "call", unpacker);
+        } else {
+          ByteList byteList = new ByteList(readBytes(size), runtime.getEncodingService().getAscii8bitEncoding());
+          return proc.callMethod(runtime.getCurrentContext(), "call", runtime.newString(byteList));
+        }
       }
     }
 
     if (this.allowUnknownExt) {
-      return ExtensionValue.newExtensionValue(runtime, type, payload);
+      return ExtensionValue.newExtensionValue(runtime, type, readBytes(size));
     }
 
     throw runtime.newRaiseException(unknownExtTypeErrorClass, "unexpected extension type");