msgpack 1.4.2 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 451ed743fe37bd3726f897f4daae9e7fd9b2cb065a34cf438ca786350060e0ec
-  data.tar.gz: 87335c4b1d35f9ac88af648963b1fe60caa72059ec7e6c1b6b507027da9118d8
+  metadata.gz: bb08f890b2d9a36312600e284933cd0de93a00a6b949507bfc3d7c5c7eab1638
+  data.tar.gz: 11e536f8bde329edbc966e5e1050134350024f89f1cb911878f86fd59ff1fc79
 SHA512:
-  metadata.gz: eb519049e9d8e4b36682e4b2c2625f10172745ce74ad1cfc8e6d93468b219071ed579bcfd36b7f0e1210eeb177197b435da2a028f46d2c71c73a3bcf2b53dcf8
-  data.tar.gz: e8c4765ac836226f5a2ce68a62194f112fc77bcdd9744d39040ff2db5a2c27d876348fb6002110c00b122bccdff2d539223d82ff73ea0bcb4cf1a3d9fb62defe
+  metadata.gz: 9491cdf494d1826cce7f5e420d73a65ce88c029e060338d9c28fb1e20e4d01dc71fdb46b91859d025bbcc8f5a0471100b171cb9b3bac5fed6f0f0715fd76abb2
+  data.tar.gz: 8d0b4db4f93ebf98063f275c0629648ab9316c7cc3cd81738f7f083c6ed4c47fc647c2fc5df1308089b96c36d7e13944028e75bae7e5897b5bb357701c0bcfd9

data/.github/workflows/ci.yaml

@@ -0,0 +1,57 @@
+name: ci
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - master
+      - main
+      - 'release-*'
+
+jobs:
+  mri:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, macos, windows]
+        ruby: ['2.4', '2.5', '2.6', '2.7', '3.0', '3.1']
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true # 'bundle install' and cache
+      - run: bundle exec rake
+
+  other:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu]
+        ruby: ['jruby-9.2.19.0', 'jruby-9.3.3.0', 'truffleruby']
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true # 'bundle install' and cache
+      - run: bundle exec rake spec
+
+  head-versions:
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu]
+        ruby: ['ruby-head', 'jruby-head']
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true # 'bundle install' and cache
+      - run: bundle exec rake spec || echo "failed, but ignore it"

data/ChangeLog

@@ -1,3 +1,63 @@
+2022-09-30 1.6.0:
+
+* Fix a potential use-after-free bug in Buffer_free when accessing a packer or unpacker buffer. 
+* `old-style-definition` compilation warnings.
+* Restore zero-copy buffer feed when provided a Ruby string. This was accidentally broken in 1.5.4.
+* Provide implementations for `ObjectSpace.memsize`. Message pack objects now properly report their size to Ruby.
+* Fix an endianess bug on Windows platform.
+
+2022-08-23 1.5.6:
+
+* No actual code change, just re-release the `java` version properly.
+
+2022-08-22 1.5.5:
+
+* Fix a segfault when GC triggers inside a recursive extension.
+
+2022-07-25 1.5.4:
+
+* Fix a segfault when deserializing empty symbol (`:""`).
+* Improve compilation flags to not strip debug symbols.
+
+2022-05-30 version 1.5.3:
+
+* Fix deduplication of empty strings when using the `freeze: true` option.
+* Use `rb_hash_new_capa` when available (Ruby 3.2) for improved performance when parsing large hashes.
+
+2022-05-27 version 1.5.2:
+
+* Fix bug about unpacking ext type objects with the recursive option
+
+2022-04-07 version 1.5.1:
+
+* Fix bug about packing/unpacking ext type objects with the recursive option
+
+2022-04-06 version 1.5.0:
+
+* Add recursive option on Factory#register_type to operate Packer/Unpacker manually
+* Add oversized_integer_extension option on Factory#register_type to pack/unpack bigint using ext types
+* Add Factory#pool method and Factory::Pool class to provide pooled Packer and Unpacker instances
+
+2022-02-15 version 1.4.5:
+
+* Fix to create UTF-8 Symbol keys when symbolize_keys: true
+* Fix to assume Symbols as US-ASCII or UTF-8
+* Optimize Packer/Unpacker initialization
+* Optimize extension class lookup
+* Rename Packer#clear as Packer#reset (#clear is still available as an alias)
+
+2022-01-22 version 1.4.4:
+
+* Specify the build option --platform=8 for older Java platforms
+
+2022-01-20 version 1.4.3:
+
+* Optimize serialization/deserialization of Symbols
+* Support registering ext types for objects of subclasses of primitive types (like Hash)
+* Add optimized_symbols_parsing option to Factory#register_type on MRI implementation
+* Optimize to deduplicate Hash keys on JRuby
+* Support JRuby 9.3 (and drop 9.1)
+
 2021-02-01 version 1.4.2:
 
 * Add the required Ruby version (>= 2.4) to avoid compilation errors on older Ruby runtimes

data/README.md

@@ -40,7 +40,7 @@ or build msgpack-ruby and install:
 MessagePack for Ruby should run on x86, ARM, PowerPC, SPARC and other CPU architectures.
 
 And it works with MRI (CRuby) and Rubinius.
-Patches to improve portability is highly welcomed.
+Patches to improve portability are highly welcomed.
 
 
 ## Serializing objects
@@ -51,6 +51,7 @@ Use `MessagePack.pack` or `to_msgpack`:
 require 'msgpack'
 msg = MessagePack.pack(obj)  # or
 msg = obj.to_msgpack
+File.binwrite('mydata.msgpack', msg)
 ```
 
 ### Streaming serialization
@@ -71,6 +72,7 @@ Use `MessagePack.unpack`:
 
 ```ruby
 require 'msgpack'
+msg = File.binread('mydata.msgpack')
 obj = MessagePack.unpack(msg)
 ```
 
@@ -187,6 +189,28 @@ MessagePack::DefaultFactory.register_type(0x03, MyClass3)
 MessagePack.unpack(data_with_ext_typeid_03) #=> MyClass3 instance
 ```
 
+Alternatively, extension types can call the packer or unpacker recursively to generate the extension data:
+
+```ruby
+Point = Struct.new(:x, :y)
+factory = MessagePack::Factory.new
+factory.register_type(
+  0x01,
+  Point,
+  packer: ->(point, packer) {
+    packer.write(point.x)
+    packer.write(point.y)
+  },
+  unpacker: ->(unpacker) {
+    x = unpacker.read
+    y = unpacker.read
+    Point.new(x, y)
+  },
+  recursive: true,
+)
+factory.load(factory.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
+```
+
 ## Buffer API
 
 MessagePack for Ruby provides a buffer API so that you can read or write data by hand, not via Packer or Unpacker API.

data/Rakefile

@@ -34,8 +34,7 @@ if RUBY_PLATFORM =~ /java/
     jars = ["#{jruby_home}/lib/jruby.jar"]
     ext.classpath = jars.map { |x| File.expand_path(x) }.join(':')
     ext.lib_dir = File.join(*['lib', 'msgpack', ENV['FAT_DIR']].compact)
-    ext.source_version = '1.6'
-    ext.target_version = '1.6'
+    ext.release = '8'
   end
 else
   require 'rake/extensiontask'

data/bench/bench.rb

@@ -0,0 +1,78 @@
+# % bundle install
+# % bundle exec ruby bench/bench.rb
+
+require 'msgpack'
+
+require 'benchmark/ips'
+
+object_plain = {
+  'message' => '127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"'
+}
+
+data_plain = MessagePack.pack(object_plain)
+
+object_structured = {
+  'remote_host' => '127.0.0.1',
+  'remote_user' => '-',
+  'date' => '10/Oct/2000:13:55:36 -0700',
+  'request' => 'GET /apache_pb.gif HTTP/1.0',
+  'method' => 'GET',
+  'path' => '/apache_pb.gif',
+  'protocol' => 'HTTP/1.0',
+  'status' => 200,
+  'bytes' => 2326,
+  'referer' => 'http://www.example.com/start.html',
+  'agent' => 'Mozilla/4.08 [en] (Win98; I ;Nav)',
+}
+
+data_structured = MessagePack.pack(object_structured)
+
+class Extended
+  def to_msgpack_ext
+    MessagePack.pack({})
+  end
+
+  def self.from_msgpack_ext(data)
+    MessagePack.unpack(data)
+    Extended.new
+  end
+end
+
+object_extended = {
+  'extended' => Extended.new
+}
+
+extended_packer = MessagePack::Packer.new
+extended_packer.register_type(0x00, Extended, :to_msgpack_ext)
+data_extended = extended_packer.pack(object_extended).to_s
+
+Benchmark.ips do |x|
+  x.report('pack-plain') do
+    MessagePack.pack(object_plain)
+  end
+
+  x.report('pack-structured') do
+    MessagePack.pack(object_structured)
+  end
+
+  x.report('pack-extended') do
+    packer = MessagePack::Packer.new
+    packer.register_type(0x00, Extended, :to_msgpack_ext)
+    packer.pack(object_extended).to_s
+  end
+
+  x.report('unpack-plain') do
+    MessagePack.unpack(data_plain)
+  end
+
+  x.report('unpack-structured') do
+    MessagePack.unpack(data_structured)
+  end
+
+  x.report('unpack-extended') do
+    unpacker = MessagePack::Unpacker.new
+    unpacker.register_type(0x00, Extended, :from_msgpack_ext)
+    unpacker.feed data_extended
+    unpacker.read
+  end
+end

data/bin/console

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "msgpack"
+
+require "irb"
+IRB.start(__FILE__)

data/doclib/msgpack/factory.rb

@@ -31,7 +31,7 @@ module MessagePack
     #
     # See Packer#initialize for supported options.
     #
-    def dump(obj, options={})
+    def dump(obj, options=nil)
     end
     alias pack dump
 
@@ -57,13 +57,13 @@ module MessagePack
     #
     # See Unpacker#initialize for supported options.
     #
-    def load(data, options={})
+    def load(data, options=nil)
     end
     alias unpack load
 
     #
     # Register a type and Class to be registered for packer and/or unpacker.
-    # If options are not speicified, factory will use :to_msgpack_ext for packer, and
+    # If options are not specified, factory will use :to_msgpack_ext for packer, and
     # :from_msgpack_ext for unpacker.
     #
     # @param type [Fixnum] type id of registered Class (0-127)
@@ -75,6 +75,8 @@ module MessagePack
     #
     # * *:packer* specify symbol or proc object for packer
     # * *:unpacker* specify symbol or proc object for unpacker
+    # * *:optimized_symbols_parsing* specify true to use the optimized symbols parsing (not supported on JRuby now)
+    # * *recursive* specify true to receive the packer or unpacker as argument to generate the extension body manually.
     #
     def register_type(type, klass, options={})
     end
@@ -97,5 +99,47 @@ module MessagePack
     #
     def type_registered?(klass_or_type, selector=:both)
     end
+
+    #
+    # Creates a MessagePack::PooledFactory instance of the given size.
+    #
+    # PooledFactory keeps Packer and Unpacker instance in a pool for improved performance.
+    # Note that the size defines how many instances are kept in cache, not the maximum of instances
+    # that can be created. If the pool limit is reached, a new instance is created anyway.
+    #
+    # @param size [Fixnum] specify how many Packer and Unpacker to keep in cache (default 1)
+    # @param options [Hash] Combined options for Packer and Unpacker. See Packer#initialize and Unpacker#initialize
+    #                       for supported options.
+    def pool(size=1, **options)
+    end
+
+    class Pool
+      #
+      # Deserializes an object from the string or io and returns it.
+      #
+      # If there're not enough data to deserialize one object, this method raises EOFError.
+      # If data format is invalid, this method raises MessagePack::MalformedFormatError.
+      # If the object nests too deeply, this method raises MessagePack::StackError.
+      #
+      # @param data [String]
+      # @return [Object] deserialized object
+      #
+      # See Unpacker#initialize for supported options.
+      #
+      def load(data)
+      end
+
+      #
+      # Serialize the passed value
+      #
+      # If it could not serialize the object, it raises
+      # NoMethodError: undefined method `to_msgpack' for #<the_object>.
+      #
+      # @param obj [Object] object to serialize
+      # @return [String] serialized object
+      #
+      def dump(object)
+      end
+    end
   end
 end

data/doclib/msgpack/packer.rb

@@ -14,7 +14,7 @@ module MessagePack
     # @overload initialize(io, options={})
     #   @param io [IO]
     #   @param options [Hash]
-    #   This packer writes serialzied objects into the IO when the internal buffer is filled.
+    #   This packer writes serialized objects into the IO when the internal buffer is filled.
     #   _io_ must respond to write(string) or append(string) method.
     #
     # Supported options:
@@ -33,12 +33,12 @@ module MessagePack
     #
     # @overload register_type(type, klass, &block)
     #   @param type [Fixnum] type id (0-127) user defined type id for specified Class
-    #   @param klass [Class] Class to be serialized with speicifed type id
+    #   @param klass [Class] Class to be serialized with specified type id
     #   @yieldparam object [Object] object to be serialized
     #
     # @overload register_type(type, klass, method_name)
     #   @param type [Fixnum] type id (0-127) user defined type id for specified Class
-    #   @param klass [Class] Class to be serialized with speicifed type id
+    #   @param klass [Class] Class to be serialized with specified type id
     #   @param method_name [Symbol] method which returns bytes of serialized representation
     #
     # @return nil
@@ -155,8 +155,9 @@ module MessagePack
     #
     # @return nil
     #
-    def clear
+    def reset
     end
+    alias clear reset
 
     #
     # Returns size of the internal buffer. Same as buffer.size.

data/doclib/msgpack/unpacker.rb

@@ -36,7 +36,7 @@ module MessagePack
     #
     # @overload register_type(type, klass, class_method_name)
     #   @param type [Fixnum] type id (0-127) user defined type id for specified Class
-    #   @param klass [Class] Class to be serialized with speicifed type id
+    #   @param klass [Class] Class to be serialized with specified type id
     #   @param class_method_name [Symbol] class method which returns an instance object
     #
     # @return nil
@@ -149,7 +149,7 @@ module MessagePack
     #
     # It repeats until the io or internal buffer does not include any complete objects.
     #
-    # If the an IO is set, it repeats to read data from the IO when the buffer
+    # If an IO is set, it repeats to read data from the IO when the buffer
     # becomes empty until the IO raises EOFError.
     #
     # This method could raise same errors with _read_ excepting EOFError.

data/ext/java/org/msgpack/jruby/Buffer.java

@@ -21,6 +21,7 @@ import org.jcodings.Encoding;
 
 @JRubyClass(name="MessagePack::Buffer")
 public class Buffer extends RubyObject {
+  private static final long serialVersionUID = 8441244627425629412L;
   private IRubyObject io;
   private ByteBuffer buffer;
   private boolean writeMode;
@@ -49,7 +50,7 @@ public class Buffer extends RubyObject {
     }
     this.buffer = ByteBuffer.allocate(CACHE_LINE_SIZE - ARRAY_HEADER_SIZE);
     this.writeMode = true;
-    this.binaryEncoding = ctx.getRuntime().getEncodingService().getAscii8bitEncoding();
+    this.binaryEncoding = ctx.runtime.getEncodingService().getAscii8bitEncoding();
     return this;
   }
 
@@ -87,17 +88,17 @@ public class Buffer extends RubyObject {
       writeMode = true;
     }
     buffer.clear();
-    return ctx.getRuntime().getNil();
+    return ctx.runtime.getNil();
   }
 
   @JRubyMethod(name = "size")
   public IRubyObject size(ThreadContext ctx) {
-    return ctx.getRuntime().newFixnum(rawSize());
+    return ctx.runtime.newFixnum(rawSize());
   }
 
   @JRubyMethod(name = "empty?")
   public IRubyObject isEmpty(ThreadContext ctx) {
-    return rawSize() == 0 ? ctx.getRuntime().getTrue() : ctx.getRuntime().getFalse();
+    return rawSize() == 0 ? ctx.runtime.getTrue() : ctx.runtime.getFalse();
   }
 
   private IRubyObject bufferWrite(ThreadContext ctx, IRubyObject str) {
@@ -105,7 +106,7 @@ public class Buffer extends RubyObject {
     int length = bytes.length();
     ensureRemainingCapacity(length);
     buffer.put(bytes.unsafeBytes(), bytes.begin(), length);
-    return ctx.getRuntime().newFixnum(length);
+    return ctx.runtime.newFixnum(length);
 
   }
 
@@ -131,19 +132,19 @@ public class Buffer extends RubyObject {
       length = (int) args[0].convertToInteger().getLongValue();
     }
     if (raiseOnUnderflow && rawSize() < length) {
-      throw ctx.getRuntime().newEOFError();
+      throw ctx.runtime.newEOFError();
     }
     int readLength = Math.min(length, rawSize());
     if (readLength == 0 && length > 0) {
-      return ctx.getRuntime().getNil();
+      return ctx.runtime.getNil();
     } else if (readLength == 0) {
-      return ctx.getRuntime().newString();
+      return ctx.runtime.newString();
     } else {
       ensureReadMode();
       byte[] bytes = new byte[readLength];
       buffer.get(bytes);
       ByteList byteList = new ByteList(bytes, binaryEncoding);
-      return ctx.getRuntime().newString(byteList);
+      return ctx.runtime.newString(byteList);
     }
   }
 
@@ -161,12 +162,12 @@ public class Buffer extends RubyObject {
     feed(ctx);
     int length = (int) _length.convertToInteger().getLongValue();
     if (raiseOnUnderflow && rawSize() < length) {
-      throw ctx.getRuntime().newEOFError();
+      throw ctx.runtime.newEOFError();
     }
     ensureReadMode();
     int skipLength = Math.min(length, rawSize());
     buffer.position(buffer.position() + skipLength);
-    return ctx.getRuntime().newFixnum(skipLength);
+    return ctx.runtime.newFixnum(skipLength);
   }
 
   @JRubyMethod(name = "skip")
@@ -188,23 +189,23 @@ public class Buffer extends RubyObject {
     ensureReadMode();
     int length = buffer.limit() - buffer.position();
     ByteList str = new ByteList(buffer.array(), buffer.position(), length, binaryEncoding, true);
-    return ctx.getRuntime().newString(str);
+    return ctx.runtime.newString(str);
   }
 
   @JRubyMethod(name = "to_a")
   public IRubyObject toA(ThreadContext ctx) {
-    return ctx.getRuntime().newArray(toS(ctx));
+    return ctx.runtime.newArray(toS(ctx));
   }
 
   @JRubyMethod(name = "io")
   public IRubyObject getIo(ThreadContext ctx) {
-    return io == null ? ctx.getRuntime().getNil() : io;
+    return io == null ? ctx.runtime.getNil() : io;
   }
 
   @JRubyMethod(name = "flush")
   public IRubyObject flush(ThreadContext ctx) {
     if (io == null) {
-      return ctx.getRuntime().getNil();
+      return ctx.runtime.getNil();
     } else {
       return io.callMethod(ctx, "flush");
     }
@@ -213,7 +214,7 @@ public class Buffer extends RubyObject {
   @JRubyMethod(name = "close")
   public IRubyObject close(ThreadContext ctx) {
     if (io == null) {
-      return ctx.getRuntime().getNil();
+      return ctx.runtime.getNil();
     } else {
       return io.callMethod(ctx, "close");
     }
@@ -223,4 +224,10 @@ public class Buffer extends RubyObject {
   public IRubyObject writeTo(ThreadContext ctx, IRubyObject io) {
     return io.callMethod(ctx, "write", readCommon(ctx, null, false));
   }
+
+  public ByteList getBytes() {
+    byte[] bytes = new byte[rawSize()];
+    buffer.get(bytes);
+    return new ByteList(bytes, binaryEncoding);
+  }
 }

data/ext/java/org/msgpack/jruby/Decoder.java

@@ -14,6 +14,7 @@ import org.jruby.RubyBignum;
 import org.jruby.RubyString;
 import org.jruby.RubyArray;
 import org.jruby.RubyHash;
+import org.jruby.RubyInteger;
 import org.jruby.exceptions.RaiseException;
 import org.jruby.runtime.builtin.IRubyObject;
 import org.jruby.util.ByteList;
@@ -35,7 +36,7 @@ public class Decoder implements Iterator<IRubyObject> {
   private final RubyClass unexpectedTypeErrorClass;
   private final RubyClass unknownExtTypeErrorClass;
 
-  private ExtensionRegistry registry;
+  private Unpacker unpacker;
   private ByteBuffer buffer;
   private boolean symbolizeKeys;
   private boolean freeze;
@@ -45,29 +46,29 @@ public class Decoder implements Iterator<IRubyObject> {
     this(runtime, null, new byte[] {}, 0, 0, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry) {
-    this(runtime, registry, new byte[] {}, 0, 0, false, false, false);
+  public Decoder(Ruby runtime, Unpacker unpacker) {
+    this(runtime, unpacker, new byte[] {}, 0, 0, false, false, false);
   }
 
   public Decoder(Ruby runtime, byte[] bytes) {
     this(runtime, null, bytes, 0, bytes.length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes) {
-    this(runtime, registry, bytes, 0, bytes.length, false, false, false);
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes) {
+    this(runtime, unpacker, bytes, 0, bytes.length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
-    this(runtime, registry, bytes, 0, bytes.length, symbolizeKeys, freeze, allowUnknownExt);
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
+    this(runtime, unpacker, bytes, 0, bytes.length, symbolizeKeys, freeze, allowUnknownExt);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length) {
-    this(runtime, registry, bytes, offset, length, false, false, false);
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes, int offset, int length) {
+    this(runtime, unpacker, bytes, offset, length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
+  public Decoder(Ruby runtime, Unpacker unpacker, byte[] bytes, int offset, int length, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
     this.runtime = runtime;
-    this.registry = registry;
+    this.unpacker = unpacker;
     this.symbolizeKeys = symbolizeKeys;
     this.freeze = freeze;
     this.allowUnknownExt = allowUnknownExt;
@@ -122,7 +123,6 @@ public class Decoder implements Iterator<IRubyObject> {
     ByteList byteList = new ByteList(bytes, encoding);
     RubyString string = runtime.newString(byteList);
     if (this.freeze) {
-      string.setFrozen(true);
       string = runtime.freezeAndDedupString(string);
     }
     return string;
@@ -140,9 +140,14 @@ public class Decoder implements Iterator<IRubyObject> {
     RubyHash hash = RubyHash.newHash(runtime);
     for (int i = 0; i < size; i++) {
       IRubyObject key = next();
-      if (this.symbolizeKeys && key instanceof RubyString) {
+      if (key instanceof RubyString) {
+        if (this.symbolizeKeys) {
           key = ((RubyString) key).intern();
+        } else {
+          key = runtime.freezeAndDedupString((RubyString) key);
+        }
       }
+
       hash.fastASet(key, next());
     }
     return hash;
@@ -150,18 +155,21 @@ public class Decoder implements Iterator<IRubyObject> {
 
   private IRubyObject consumeExtension(int size) {
     int type = buffer.get();
-    byte[] payload = readBytes(size);
-
-    if (registry != null) {
-      IRubyObject proc = registry.lookupUnpackerByTypeId(type);
-      if (proc != null) {
-        ByteList byteList = new ByteList(payload, runtime.getEncodingService().getAscii8bitEncoding());
-        return proc.callMethod(runtime.getCurrentContext(), "call", runtime.newString(byteList));
+    if (unpacker != null) {
+      ExtensionRegistry.ExtensionEntry entry = unpacker.lookupExtensionByTypeId(type);
+      if (entry != null) {
+        IRubyObject proc = entry.getUnpackerProc();
+        if (entry.isRecursive()) {
+          return proc.callMethod(runtime.getCurrentContext(), "call", unpacker);
+        } else {
+          ByteList byteList = new ByteList(readBytes(size), runtime.getEncodingService().getAscii8bitEncoding());
+          return proc.callMethod(runtime.getCurrentContext(), "call", runtime.newString(byteList));
+        }
       }
     }
 
     if (this.allowUnknownExt) {
-      return ExtensionValue.newExtensionValue(runtime, type, payload);
+      return ExtensionValue.newExtensionValue(runtime, type, readBytes(size));
     }
 
     throw runtime.newRaiseException(unknownExtTypeErrorClass, "unexpected extension type");