msgpack 1.3.3 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 40d206a80a2ab99957b78f2d054417dc273933da4accf5f39a25ff4b16ec30ce
-  data.tar.gz: e39c06187017d057a44e71d3690662455934c2c1acafc667d49e91aaf626fbd9
+  metadata.gz: 2f5b1af6b3a51f5ccc6bcf67c94c1fc6193b02fe01b123e2cfb06a6df9607116
+  data.tar.gz: cc057f24e1ffa4cdc3e331499eb04de4c2383b0657dcf0baeba08300fd20862e
 SHA512:
-  metadata.gz: d20c8da71f90435f264c0e1c9c90e4a5c8c09c014d33076149e6e5faa3b849ac5192d2f086b917f09eb199bcfd8cc748093d6f59e3f5fcb7c228199f549c23c5
-  data.tar.gz: dd5c662d5abbf45901d59a7400f2b92a6c296b2298035fecc55a31f82b19b4e9206ca3b0a7cab090d9cc9768bea015b74ad9bf5a997a763eb6ba3387b6a7fc30
+  metadata.gz: 3eb06321a534ca9b16e321cc4a71458532578dafe7967314a662223b1fbf4aa93449c98177fa982aa532ce3732ddda4a6d497704df0e9c874da07f378c73595c
+  data.tar.gz: 8e540755e3db9e21d7dfa4354854e8b0486f5a1bbf82c3994c6095022205f7873153d364df9310d8072c481de38ca2b4c3e088e4221c3451ceb9438312489419

data/ChangeLog

@@ -1,3 +1,102 @@
+2023-07-18 1.7.2:
+
+* Fix a potential GC bug when packing data using recursive extensions and buffers containing over 512KkiB of data (See #341).
+* Fix a regression where feeding an empty string to an Unpacker would be considered like the end of the buffer.
+
+2023-05-19 1.7.1:
+
+* Fix JRuby 9.4 compatibility.
+* Fix compilation on older compilers (gcc 4.x).
+* Fix an infinite recursion issue when registering a Symbol type with a `nil` packer.
+
+2023-03-29 1.7.0:
+
+* Fix a possible double-free issue when GC triggers inside `_msgpack_rmem_alloc2`.
+* `Unpacker#feed` now always directly read in provided strings instead of copying content in its buffer.
+* `Unpacker#feed` is now an alias of `Unpacker#feed_reference`.
+* Implement `Factory::Pool#unpacker` and `Factory::Pool#packer` to allow for more precise serialization.
+* Require Ruby 2.5+.
+
+2023-03-03 1.6.1:
+
+* Undefine `#clone` and `#dup` on `MessagePack::Buffer`, `MessagePack::Packer` and `MessagePack::Unpacker`.
+  These methods were never intended, and using them could cause leaks or crashes or worse.
+* Fix a possible GC crash when GC trigger inside `MessagePack::Buffer.new` (#314).
+
+2022-09-30 1.6.0:
+
+* Fix a potential use-after-free bug in Buffer_free when accessing a packer or unpacker buffer. 
+* `old-style-definition` compilation warnings.
+* Restore zero-copy buffer feed when provided a Ruby string. This was accidentally broken in 1.5.4.
+* Provide implementations for `ObjectSpace.memsize`. Message pack objects now properly report their size to Ruby.
+* Fix an endianess bug on Windows platform.
+
+2022-08-23 1.5.6:
+
+* No actual code change, just re-release the `java` version properly.
+
+2022-08-22 1.5.5:
+
+* Fix a segfault when GC triggers inside a recursive extension.
+
+2022-07-25 1.5.4:
+
+* Fix a segfault when deserializing empty symbol (`:""`).
+* Improve compilation flags to not strip debug symbols.
+
+2022-05-30 version 1.5.3:
+
+* Fix deduplication of empty strings when using the `freeze: true` option.
+* Use `rb_hash_new_capa` when available (Ruby 3.2) for improved performance when parsing large hashes.
+
+2022-05-27 version 1.5.2:
+
+* Fix bug about unpacking ext type objects with the recursive option
+
+2022-04-07 version 1.5.1:
+
+* Fix bug about packing/unpacking ext type objects with the recursive option
+
+2022-04-06 version 1.5.0:
+
+* Add recursive option on Factory#register_type to operate Packer/Unpacker manually
+* Add oversized_integer_extension option on Factory#register_type to pack/unpack bigint using ext types
+* Add Factory#pool method and Factory::Pool class to provide pooled Packer and Unpacker instances
+
+2022-02-15 version 1.4.5:
+
+* Fix to create UTF-8 Symbol keys when symbolize_keys: true
+* Fix to assume Symbols as US-ASCII or UTF-8
+* Optimize Packer/Unpacker initialization
+* Optimize extension class lookup
+* Rename Packer#clear as Packer#reset (#clear is still available as an alias)
+
+2022-01-22 version 1.4.4:
+
+* Specify the build option --platform=8 for older Java platforms
+
+2022-01-20 version 1.4.3:
+
+* Optimize serialization/deserialization of Symbols
+* Support registering ext types for objects of subclasses of primitive types (like Hash)
+* Add optimized_symbols_parsing option to Factory#register_type on MRI implementation
+* Optimize to deduplicate Hash keys on JRuby
+* Support JRuby 9.3 (and drop 9.1)
+
+2021-02-01 version 1.4.2:
+
+* Add the required Ruby version (>= 2.4) to avoid compilation errors on older Ruby runtimes
+* Drop the support of old Ruby versions explicitly (1.8, 1.9, 2.0, 2.1, 2.2, 2.3)
+
+2021-01-27 version 1.4.1:
+
+* Bugfix about the wrong string encoding longer than 256 bytes (#200)
+
+2021-01-27 version 1.4.0:
+
+* Introduce the optimization to use frozen/deduped keys for map objects
+* Stop releasing fat gem (pre-built binaries) for mswin32 arch environments
+
 2020-02-05 version 1.3.3:
 
 * Hotfix release for Windows environments: 1.3.2 missed including binaries

data/README.md

@@ -0,0 +1,293 @@
+# MessagePack
+
+[MessagePack](http://msgpack.org) is an efficient binary serialization format.
+It lets you exchange data among multiple languages like JSON but it's faster and smaller.
+For example, small integers (like flags or error code) are encoded into a single byte,
+and typical short strings only require an extra byte in addition to the strings themselves.
+
+If you ever wished to use JSON for convenience (storing an image with metadata) but could
+not for technical reasons (binary data, size, speed...), MessagePack is a perfect replacement.
+
+    require 'msgpack'
+    msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
+    MessagePack.unpack(msg)   #=> [1,2,3]
+
+Use RubyGems to install:
+
+    gem install msgpack
+
+or build msgpack-ruby and install:
+
+    bundle
+    rake
+    gem install --local pkg/msgpack
+
+
+## Use cases
+
+* Create REST API returing MessagePack using Rails + [RABL](https://github.com/nesquena/rabl)
+* Store objects efficiently serialized by msgpack on memcached or Redis
+  * In fact Redis supports msgpack in [EVAL-scripts](http://redis.io/commands/eval)
+* Upload data in efficient format from mobile devices such as smartphones
+  * MessagePack works on iPhone/iPad and Android. See also [Objective-C](https://github.com/msgpack/msgpack-objectivec) and [Java](https://github.com/msgpack/msgpack-java) implementations
+* Design a portable protocol to communicate with embedded devices
+  * Check also [Fluentd](http://fluentd.org/) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
+* Exchange objects between software components written in different languages
+  * You'll need a flexible but efficient format so that components exchange objects while keeping compatibility
+
+## Portability
+
+MessagePack for Ruby should run on x86, ARM, PowerPC, SPARC and other CPU architectures.
+
+And it works with MRI (CRuby) and Rubinius.
+Patches to improve portability are highly welcomed.
+
+
+## Serializing objects
+
+Use `MessagePack.pack` or `to_msgpack`:
+
+```ruby
+require 'msgpack'
+msg = MessagePack.pack(obj)  # or
+msg = obj.to_msgpack
+File.binwrite('mydata.msgpack', msg)
+```
+
+### Streaming serialization
+
+Packer provides advanced API to serialize objects in streaming style:
+
+```ruby
+# serialize a 2-element array [e1, e2]
+pk = MessagePack::Packer.new(io)
+pk.write_array_header(2).write(e1).write(e2).flush
+```
+
+See [API reference](http://ruby.msgpack.org/MessagePack/Packer.html) for details.
+
+## Deserializing objects
+
+Use `MessagePack.unpack`:
+
+```ruby
+require 'msgpack'
+msg = File.binread('mydata.msgpack')
+obj = MessagePack.unpack(msg)
+```
+
+### Streaming deserialization
+
+Unpacker provides advanced API to deserialize objects in streaming style:
+
+```ruby
+# deserialize objects from an IO
+u = MessagePack::Unpacker.new(io)
+u.each do |obj|
+  # ...
+end
+```
+
+or event-driven style which works well with EventMachine:
+
+```ruby
+# event-driven deserialization
+def on_read(data)
+  @u ||= MessagePack::Unpacker.new
+  @u.feed_each(data) {|obj|
+     # ...
+  }
+end
+```
+
+See [API reference](http://ruby.msgpack.org/MessagePack/Unpacker.html) for details.
+
+## Serializing and deserializing symbols
+
+By default, symbols are serialized as strings:
+
+```ruby
+packed = :symbol.to_msgpack     # => "\xA6symbol"
+MessagePack.unpack(packed)      # => "symbol"
+```
+
+This can be customized by registering an extension type for them:
+
+```ruby
+MessagePack::DefaultFactory.register_type(0x00, Symbol)
+
+# symbols now survive round trips
+packed = :symbol.to_msgpack     # => "\xc7\x06\x00symbol"
+MessagePack.unpack(packed)      # => :symbol
+```
+
+The extension type for symbols is configurable like any other extension type.
+For example, to customize how symbols are packed you can just redefine
+Symbol#to_msgpack_ext. Doing this gives you an option to prevent symbols from
+being serialized altogether by throwing an exception:
+
+```ruby
+class Symbol
+    def to_msgpack_ext
+        raise "Serialization of symbols prohibited"
+    end
+end
+
+MessagePack::DefaultFactory.register_type(0x00, Symbol)
+
+[1, :symbol, 'string'].to_msgpack  # => RuntimeError: Serialization of symbols prohibited
+```
+
+## Serializing and deserializing Time instances
+
+There are the timestamp extension type in MessagePack,
+but it is not registered by default.
+
+To map Ruby's Time to MessagePack's timestamp for the default factory:
+
+```ruby
+MessagePack::DefaultFactory.register_type(
+  MessagePack::Timestamp::TYPE, # or just -1
+  Time,
+  packer: MessagePack::Time::Packer,
+  unpacker: MessagePack::Time::Unpacker
+)
+```
+
+See [API reference](http://ruby.msgpack.org/) for details.
+
+## Extension Types
+
+Packer and Unpacker support [Extension types of MessagePack](https://github.com/msgpack/msgpack/blob/master/spec.md#types-extension-type).
+
+```ruby
+# register how to serialize custom class at first
+pk = MessagePack::Packer.new(io)
+pk.register_type(0x01, MyClass1, :to_msgpack_ext) # equal to pk.register_type(0x01, MyClass)
+pk.register_type(0x02, MyClass2){|obj| obj.how_to_serialize() } # blocks also available
+
+# almost same API for unpacker
+uk = MessagePack::Unpacker.new()
+uk.register_type(0x01, MyClass1, :from_msgpack_ext)
+uk.register_type(0x02){|data| MyClass2.create_from_serialized_data(data) }
+```
+
+`MessagePack::Factory` is to create packer and unpacker which have same extension types.
+
+```ruby
+factory = MessagePack::Factory.new
+factory.register_type(0x01, MyClass1) # same with next line
+factory.register_type(0x01, MyClass1, packer: :to_msgpack_ext, unpacker: :from_msgpack_ext)
+pk = factory.packer(options_for_packer)
+uk = factory.unpacker(options_for_unpacker)
+```
+
+For `MessagePack.pack` and `MessagePack.unpack`, default packer/unpacker refer `MessagePack::DefaultFactory`. Call `MessagePack::DefaultFactory.register_type` to enable types process globally.
+
+```ruby
+MessagePack::DefaultFactory.register_type(0x03, MyClass3)
+MessagePack.unpack(data_with_ext_typeid_03) #=> MyClass3 instance
+```
+
+Alternatively, extension types can call the packer or unpacker recursively to generate the extension data:
+
+```ruby
+Point = Struct.new(:x, :y)
+factory = MessagePack::Factory.new
+factory.register_type(
+  0x01,
+  Point,
+  packer: ->(point, packer) {
+    packer.write(point.x)
+    packer.write(point.y)
+  },
+  unpacker: ->(unpacker) {
+    x = unpacker.read
+    y = unpacker.read
+    Point.new(x, y)
+  },
+  recursive: true,
+)
+factory.load(factory.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
+```
+
+## Pooling
+
+Creating `Packer` and `Unpacker` objects is expensive. For best performance it is preferable to re-use these objects.
+
+`MessagePack::Factory#pool` makes that easier:
+
+```ruby
+factory = MessagePack::Factory.new
+factory.register_type(
+  0x01,
+  Point,
+  packer: ->(point, packer) {
+    packer.write(point.x)
+    packer.write(point.y)
+  },
+  unpacker: ->(unpacker) {
+    x = unpacker.read
+    y = unpacker.read
+    Point.new(x, y)
+  },
+  recursive: true,
+)
+pool = factory.pool(5) # The pool size should match the number of threads expected to use the factory concurrently.
+
+pool.load(pool.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
+```
+
+## Buffer API
+
+MessagePack for Ruby provides a buffer API so that you can read or write data by hand, not via Packer or Unpacker API.
+
+This [MessagePack::Buffer](http://ruby.msgpack.org/MessagePack/Buffer.html) is backed with a fixed-length shared memory pool which is very fast for small data (<= 4KB),
+and has zero-copy capability which significantly affects performance to handle large binary data.
+
+## How to build and run tests
+
+Before building msgpack, you need to install bundler and dependencies.
+
+    gem install bundler
+    bundle install
+
+Then, you can run the tasks as follows:
+
+### Build
+
+    bundle exec rake build
+
+### Run tests
+
+    bundle exec rake spec
+
+### Generating docs
+
+    bundle exec rake doc
+
+## How to build -java rubygems
+
+To build -java gems for JRuby, run:
+
+    rake build:java
+
+If this directory has Gemfile.lock (generated with MRI), remove it beforehand.
+
+## Updating documents
+
+Online documents (http://ruby.msgpack.org) is generated from gh-pages branch.
+Following commands update documents in gh-pages branch:
+
+    bundle exec rake doc
+    git checkout gh-pages
+    cp doc/* ./ -a
+
+## Copyright
+
+* Author
+  * Sadayuki Furuhashi <frsyuki@gmail.com>
+* Copyright
+  * Copyright (c) 2008-2015 Sadayuki Furuhashi
+* License
+  * Apache License, Version 2.0

data/ext/java/org/msgpack/jruby/Buffer.java

@@ -21,10 +21,11 @@ import org.jcodings.Encoding;
 
 @JRubyClass(name="MessagePack::Buffer")
 public class Buffer extends RubyObject {
-  private IRubyObject io;
-  private ByteBuffer buffer;
+  private static final long serialVersionUID = 8441244627425629412L;
+  private transient IRubyObject io;
+  private transient ByteBuffer buffer;
   private boolean writeMode;
-  private Encoding binaryEncoding;
+  private transient Encoding binaryEncoding;
 
   private static final int CACHE_LINE_SIZE = 64;
   private static final int ARRAY_HEADER_SIZE = 24;
@@ -49,7 +50,7 @@ public class Buffer extends RubyObject {
     }
     this.buffer = ByteBuffer.allocate(CACHE_LINE_SIZE - ARRAY_HEADER_SIZE);
     this.writeMode = true;
-    this.binaryEncoding = ctx.getRuntime().getEncodingService().getAscii8bitEncoding();
+    this.binaryEncoding = ctx.runtime.getEncodingService().getAscii8bitEncoding();
     return this;
   }
 
@@ -87,17 +88,17 @@ public class Buffer extends RubyObject {
       writeMode = true;
     }
     buffer.clear();
-    return ctx.getRuntime().getNil();
+    return ctx.runtime.getNil();
   }
 
   @JRubyMethod(name = "size")
   public IRubyObject size(ThreadContext ctx) {
-    return ctx.getRuntime().newFixnum(rawSize());
+    return ctx.runtime.newFixnum(rawSize());
   }
 
   @JRubyMethod(name = "empty?")
   public IRubyObject isEmpty(ThreadContext ctx) {
-    return rawSize() == 0 ? ctx.getRuntime().getTrue() : ctx.getRuntime().getFalse();
+    return rawSize() == 0 ? ctx.runtime.getTrue() : ctx.runtime.getFalse();
   }
 
   private IRubyObject bufferWrite(ThreadContext ctx, IRubyObject str) {
@@ -105,7 +106,7 @@ public class Buffer extends RubyObject {
     int length = bytes.length();
     ensureRemainingCapacity(length);
     buffer.put(bytes.unsafeBytes(), bytes.begin(), length);
-    return ctx.getRuntime().newFixnum(length);
+    return ctx.runtime.newFixnum(length);
 
   }
 
@@ -131,19 +132,19 @@ public class Buffer extends RubyObject {
       length = (int) args[0].convertToInteger().getLongValue();
     }
     if (raiseOnUnderflow && rawSize() < length) {
-      throw ctx.getRuntime().newEOFError();
+      throw ctx.runtime.newEOFError();
     }
     int readLength = Math.min(length, rawSize());
     if (readLength == 0 && length > 0) {
-      return ctx.getRuntime().getNil();
+      return ctx.runtime.getNil();
     } else if (readLength == 0) {
-      return ctx.getRuntime().newString();
+      return ctx.runtime.newString();
     } else {
       ensureReadMode();
       byte[] bytes = new byte[readLength];
       buffer.get(bytes);
       ByteList byteList = new ByteList(bytes, binaryEncoding);
-      return ctx.getRuntime().newString(byteList);
+      return ctx.runtime.newString(byteList);
     }
   }
 
@@ -161,12 +162,12 @@ public class Buffer extends RubyObject {
     feed(ctx);
     int length = (int) _length.convertToInteger().getLongValue();
     if (raiseOnUnderflow && rawSize() < length) {
-      throw ctx.getRuntime().newEOFError();
+      throw ctx.runtime.newEOFError();
     }
     ensureReadMode();
     int skipLength = Math.min(length, rawSize());
     buffer.position(buffer.position() + skipLength);
-    return ctx.getRuntime().newFixnum(skipLength);
+    return ctx.runtime.newFixnum(skipLength);
   }
 
   @JRubyMethod(name = "skip")
@@ -188,23 +189,23 @@ public class Buffer extends RubyObject {
     ensureReadMode();
     int length = buffer.limit() - buffer.position();
     ByteList str = new ByteList(buffer.array(), buffer.position(), length, binaryEncoding, true);
-    return ctx.getRuntime().newString(str);
+    return ctx.runtime.newString(str);
   }
 
   @JRubyMethod(name = "to_a")
   public IRubyObject toA(ThreadContext ctx) {
-    return ctx.getRuntime().newArray(toS(ctx));
+    return ctx.runtime.newArray(toS(ctx));
   }
 
   @JRubyMethod(name = "io")
   public IRubyObject getIo(ThreadContext ctx) {
-    return io == null ? ctx.getRuntime().getNil() : io;
+    return io == null ? ctx.runtime.getNil() : io;
   }
 
   @JRubyMethod(name = "flush")
   public IRubyObject flush(ThreadContext ctx) {
     if (io == null) {
-      return ctx.getRuntime().getNil();
+      return ctx.runtime.getNil();
     } else {
       return io.callMethod(ctx, "flush");
     }
@@ -213,7 +214,7 @@ public class Buffer extends RubyObject {
   @JRubyMethod(name = "close")
   public IRubyObject close(ThreadContext ctx) {
     if (io == null) {
-      return ctx.getRuntime().getNil();
+      return ctx.runtime.getNil();
     } else {
       return io.callMethod(ctx, "close");
     }
@@ -223,4 +224,10 @@ public class Buffer extends RubyObject {
   public IRubyObject writeTo(ThreadContext ctx, IRubyObject io) {
     return io.callMethod(ctx, "write", readCommon(ctx, null, false));
   }
+
+  public ByteList getBytes() {
+    byte[] bytes = new byte[rawSize()];
+    buffer.get(bytes);
+    return new ByteList(bytes, binaryEncoding);
+  }
 }

data/ext/java/org/msgpack/jruby/Decoder.java

@@ -14,6 +14,7 @@ import org.jruby.RubyBignum;
 import org.jruby.RubyString;
 import org.jruby.RubyArray;
 import org.jruby.RubyHash;
+import org.jruby.RubyInteger;
 import org.jruby.exceptions.RaiseException;
 import org.jruby.runtime.builtin.IRubyObject;
 import org.jruby.util.ByteList;
@@ -35,39 +36,41 @@ public class Decoder implements Iterator<IRubyObject> {
   private final RubyClass unexpectedTypeErrorClass;
   private final RubyClass unknownExtTypeErrorClass;
 
-  private ExtensionRegistry registry;
+  private Unpacker unpacker;
   private ByteBuffer buffer;
   private boolean symbolizeKeys;
+  private boolean freeze;
   private boolean allowUnknownExt;
 
   public Decoder(Ruby runtime) {
-    this(runtime, null, new byte[] {}, 0, 0, false, false);
+    this(runtime, null, new byte[] {}, 0, 0, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry) {
-    this(runtime, registry, new byte[] {}, 0, 0, false, false);
+  public Decoder(Ruby runtime, Unpacker unpacker) {
+    this(runtime, unpacker, new byte[] {}, 0, 0, false, false, false);
   }
 
   public Decoder(Ruby runtime, byte[] bytes) {
-    this(runtime, null, bytes, 0, bytes.length, false, false);
+    this(runtime, null, bytes, 0, bytes.length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes) {
-    this(runtime, registry, bytes, 0, bytes.length, false, false);
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes) {
+    this(runtime, unpacker, bytes, 0, bytes.length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, boolean symbolizeKeys, boolean allowUnknownExt) {
-    this(runtime, registry, bytes, 0, bytes.length, symbolizeKeys, allowUnknownExt);
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
+    this(runtime, unpacker, bytes, 0, bytes.length, symbolizeKeys, freeze, allowUnknownExt);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length) {
-    this(runtime, registry, bytes, offset, length, false, false);
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes, int offset, int length) {
+    this(runtime, unpacker, bytes, offset, length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length, boolean symbolizeKeys, boolean allowUnknownExt) {
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes, int offset, int length, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
     this.runtime = runtime;
-    this.registry = registry;
+    this.unpacker = unpacker;
     this.symbolizeKeys = symbolizeKeys;
+    this.freeze = freeze;
     this.allowUnknownExt = allowUnknownExt;
     this.binaryEncoding = runtime.getEncodingService().getAscii8bitEncoding();
     this.utf8Encoding = UTF8Encoding.INSTANCE;
@@ -118,7 +121,11 @@ public class Decoder implements Iterator<IRubyObject> {
   private IRubyObject consumeString(int size, Encoding encoding) {
     byte[] bytes = readBytes(size);
     ByteList byteList = new ByteList(bytes, encoding);
-    return runtime.newString(byteList);
+    RubyString string = runtime.newString(byteList);
+    if (this.freeze) {
+      string = runtime.freezeAndDedupString(string);
+    }
+    return string;
   }
 
   private IRubyObject consumeArray(int size) {
@@ -133,9 +140,14 @@ public class Decoder implements Iterator<IRubyObject> {
     RubyHash hash = RubyHash.newHash(runtime);
     for (int i = 0; i < size; i++) {
       IRubyObject key = next();
-      if (this.symbolizeKeys && key instanceof RubyString) {
+      if (key instanceof RubyString) {
+        if (this.symbolizeKeys) {
           key = ((RubyString) key).intern();
+        } else {
+          key = runtime.freezeAndDedupString((RubyString) key);
+        }
       }
+
       hash.fastASet(key, next());
     }
     return hash;
@@ -143,18 +155,21 @@ public class Decoder implements Iterator<IRubyObject> {
 
   private IRubyObject consumeExtension(int size) {
     int type = buffer.get();
-    byte[] payload = readBytes(size);
-
-    if (registry != null) {
-      IRubyObject proc = registry.lookupUnpackerByTypeId(type);
-      if (proc != null) {
-        ByteList byteList = new ByteList(payload, runtime.getEncodingService().getAscii8bitEncoding());
-        return proc.callMethod(runtime.getCurrentContext(), "call", runtime.newString(byteList));
+    if (unpacker != null) {
+      ExtensionRegistry.ExtensionEntry entry = unpacker.lookupExtensionByTypeId(type);
+      if (entry != null) {
+        IRubyObject proc = entry.getUnpackerProc();
+        if (entry.isRecursive()) {
+          return proc.callMethod(runtime.getCurrentContext(), "call", unpacker);
+        } else {
+          ByteList byteList = new ByteList(readBytes(size), runtime.getEncodingService().getAscii8bitEncoding());
+          return proc.callMethod(runtime.getCurrentContext(), "call", runtime.newString(byteList));
+        }
       }
     }
 
     if (this.allowUnknownExt) {
-      return ExtensionValue.newExtensionValue(runtime, type, payload);
+      return ExtensionValue.newExtensionValue(runtime, type, readBytes(size));
     }
 
     throw runtime.newRaiseException(unknownExtTypeErrorClass, "unexpected extension type");
@@ -220,6 +235,14 @@ public class Decoder implements Iterator<IRubyObject> {
 
   @Override
   public IRubyObject next() {
+    IRubyObject next = consumeNext();
+    if (freeze) {
+      next.setFrozen(true);
+    }
+    return next;
+  }
+
+  private IRubyObject consumeNext() {
     int position = buffer.position();
     try {
       byte b = buffer.get();