msgpack 1.7.1 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2caf680ff4cafade89a14da270f52528ba2a862cea237ad028ba27fa6a93e9b
-  data.tar.gz: 01d8cc906d925acbd32a88c22f6ace7f5534a0b447be1736ea58cfdf9abb85b6
+  metadata.gz: efdb772bf54b74587a6c99e9513f9c16c78bbef3e5e3c17064a4be79fd5adb7a
+  data.tar.gz: d2f74cb1115947f5337cd730b6283fac61ceeed3a9457597cc90155b3dc93d7e
 SHA512:
-  metadata.gz: 48fcd12bfb13741d88a17e1a261cd742c903ba53cb1959a88b7083c2344817d90cfa898e775c136d785fbfcdd76be14b6dc5fc8ba3638a13ecabbb2d1387dbee
-  data.tar.gz: 310f3ea573cf41b15c47bc0033121496bd8ab7a723ef3667a96a38bc286feb79536ac5aeb8d27c3c71568149f93c8f36cc9ca8e050595e27ff87762cbccfe2dd
+  metadata.gz: 92da2466eac162f0d6d319496d7af3dbbef0d76b77c3aefb4787213e3b957464603859101ae28ac9620be82feb29278b2ed601adf2fd9d9811b364c258061d4b
+  data.tar.gz: a469586178eb44bbd50abf4cb34684f6a3fd4038063a80d9b3bf7f5cb9e6a2b34c1fed853be669f0bfb356d618efff3a2eaa51b86dd34d5427787a05453b8e11

data/ChangeLog

@@ -1,3 +1,25 @@
+2025-02-06 1.8.0
+
+* Numerous small optimizations.
+* Added `key_cache` option to `Unpacker`.
+
+2024-11-11 1.7.5
+
+* Rerelease 1.7.4 with fixed java package.
+
+2024-11-11 1.7.4
+
+* Fixed a potental memory leak when recursive unpacker raise.
+
+2024-10-03 1.7.3
+
+* Limit initial containers pre-allocation to `SHRT_MAX` (32k) entries.
+
+2023-07-18 1.7.2:
+
+* Fix a potential GC bug when packing data using recursive extensions and buffers containing over 512KkiB of data (See #341).
+* Fix a regression where feeding an empty string to an Unpacker would be considered like the end of the buffer.
+
 2023-05-19 1.7.1:
 
 * Fix JRuby 9.4 compatibility.

data/README.md

@@ -8,15 +8,24 @@ and typical short strings only require an extra byte in addition to the strings
 If you ever wished to use JSON for convenience (storing an image with metadata) but could
 not for technical reasons (binary data, size, speed...), MessagePack is a perfect replacement.
 
-    require 'msgpack'
-    msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
-    MessagePack.unpack(msg)   #=> [1,2,3]
+```ruby
+require 'msgpack'
+msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
+MessagePack.unpack(msg)   #=> [1,2,3]
+```
+
+Add msgpack to your Gemfile to install with Bundler:
+
+```ruby
+# Gemfile
+gem 'msgpack'
+```
 
-Use RubyGems to install:
+Or, use RubyGems to install:
 
     gem install msgpack
 
-or build msgpack-ruby and install:
+Or, build msgpack-ruby and install from a checked-out msgpack-ruby repository:
 
     bundle
     rake
@@ -27,11 +36,11 @@ or build msgpack-ruby and install:
 
 * Create REST API returing MessagePack using Rails + [RABL](https://github.com/nesquena/rabl)
 * Store objects efficiently serialized by msgpack on memcached or Redis
-  * In fact Redis supports msgpack in [EVAL-scripts](http://redis.io/commands/eval)
+  * In fact Redis supports msgpack in [EVAL-scripts](https://redis.io/docs/latest/commands/eval/)
 * Upload data in efficient format from mobile devices such as smartphones
   * MessagePack works on iPhone/iPad and Android. See also [Objective-C](https://github.com/msgpack/msgpack-objectivec) and [Java](https://github.com/msgpack/msgpack-java) implementations
 * Design a portable protocol to communicate with embedded devices
-  * Check also [Fluentd](http://fluentd.org/) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
+  * Check also [Fluentd](https://www.fluentd.org) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
 * Exchange objects between software components written in different languages
   * You'll need a flexible but efficient format so that components exchange objects while keeping compatibility
 
@@ -128,9 +137,9 @@ being serialized altogether by throwing an exception:
 
 ```ruby
 class Symbol
-    def to_msgpack_ext
-        raise "Serialization of symbols prohibited"
-    end
+  def to_msgpack_ext
+    raise "Serialization of symbols prohibited"
+  end
 end
 
 MessagePack::DefaultFactory.register_type(0x00, Symbol)
@@ -276,8 +285,8 @@ If this directory has Gemfile.lock (generated with MRI), remove it beforehand.
 
 ## Updating documents
 
-Online documents (http://ruby.msgpack.org) is generated from gh-pages branch.
-Following commands update documents in gh-pages branch:
+Online documentation (https://ruby.msgpack.org) is generated from the gh-pages branch.
+To update documents in gh-pages branch:
 
     bundle exec rake doc
     git checkout gh-pages

data/ext/java/org/msgpack/jruby/ExtensionRegistry.java

@@ -54,8 +54,8 @@ public class ExtensionRegistry {
     return hash;
   }
 
-  public void put(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject packerArg, IRubyObject unpackerProc, IRubyObject unpackerArg) {
-    ExtensionEntry entry = new ExtensionEntry(mod, typeId, recursive, packerProc, packerArg, unpackerProc, unpackerArg);
+  public void put(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject unpackerProc) {
+    ExtensionEntry entry = new ExtensionEntry(mod, typeId, recursive, packerProc, unpackerProc);
     extensionsByModule.put(mod, entry);
     extensionsByTypeId[typeId + 128] = entry;
     extensionsByAncestor.clear();
@@ -114,18 +114,14 @@ public class ExtensionRegistry {
     private final int typeId;
     private final boolean recursive;
     private final IRubyObject packerProc;
-    private final IRubyObject packerArg;
     private final IRubyObject unpackerProc;
-    private final IRubyObject unpackerArg;
 
-    public ExtensionEntry(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject packerArg, IRubyObject unpackerProc, IRubyObject unpackerArg) {
+    public ExtensionEntry(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject unpackerProc) {
       this.mod = mod;
       this.typeId = typeId;
       this.recursive = recursive;
       this.packerProc = packerProc;
-      this.packerArg = packerArg;
       this.unpackerProc = unpackerProc;
-      this.unpackerArg = unpackerArg;
     }
 
     public RubyModule getExtensionModule() {
@@ -157,11 +153,11 @@ public class ExtensionRegistry {
     }
 
     public RubyArray<?> toPackerTuple(ThreadContext ctx) {
-      return ctx.runtime.newArray(new IRubyObject[] {ctx.runtime.newFixnum(typeId), packerProc, packerArg});
+      return ctx.runtime.newArray(new IRubyObject[] {ctx.runtime.newFixnum(typeId), packerProc});
     }
 
     public RubyArray<?> toUnpackerTuple(ThreadContext ctx) {
-      return ctx.runtime.newArray(new IRubyObject[] {mod, unpackerProc, unpackerArg});
+      return ctx.runtime.newArray(new IRubyObject[] {mod, unpackerProc});
     }
 
     public IRubyObject[] toPackerProcTypeIdPair(ThreadContext ctx) {

data/ext/java/org/msgpack/jruby/Factory.java

@@ -80,43 +80,15 @@ public class Factory extends RubyObject {
     });
   }
 
-  @JRubyMethod(name = "register_type", required = 2, optional = 1)
-  public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args) {
+  @JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
+  public IRubyObject registerTypeInternal(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject opts) {
     testFrozen("MessagePack::Factory");
 
     Ruby runtime = ctx.runtime;
-    IRubyObject type = args[0];
-    IRubyObject mod = args[1];
-
-    IRubyObject packerArg;
-    IRubyObject unpackerArg;
-
-    RubyHash options = null;
-
-    if (args.length == 2) {
-      packerArg = runtime.newSymbol("to_msgpack_ext");
-      unpackerArg = runtime.newSymbol("from_msgpack_ext");
-    } else if (args.length == 3) {
-      if (args[args.length - 1] instanceof RubyHash) {
-        options = (RubyHash) args[args.length - 1];
-        packerArg = options.fastARef(runtime.newSymbol("packer"));
-        if (packerArg != null && packerArg.isNil()) {
-          packerArg = null;
-        }
-        unpackerArg = options.fastARef(runtime.newSymbol("unpacker"));
-        if (unpackerArg != null && unpackerArg.isNil()) {
-          unpackerArg = null;
-        }
-        IRubyObject optimizedSymbolsParsingArg = options.fastARef(runtime.newSymbol("optimized_symbols_parsing"));
-        if (optimizedSymbolsParsingArg != null && optimizedSymbolsParsingArg.isTrue()) {
-          throw runtime.newArgumentError("JRuby implementation does not support the optimized_symbols_parsing option");
-        }
-      } else {
-        throw runtime.newArgumentError(String.format("expected Hash but found %s.", args[args.length - 1].getType().getName()));
-      }
-    } else {
-      throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 2..3)", 2 + args.length));
-    }
+    RubyHash options = (RubyHash) opts;
+
+    IRubyObject packerProc = options.fastARef(runtime.newSymbol("packer"));
+    IRubyObject unpackerProc = options.fastARef(runtime.newSymbol("unpacker"));
 
     long typeId = ((RubyFixnum) type).getLongValue();
     if (typeId < -128 || typeId > 127) {
@@ -128,21 +100,6 @@ public class Factory extends RubyObject {
     }
     RubyModule extModule = (RubyModule) mod;
 
-    IRubyObject packerProc = runtime.getNil();
-    IRubyObject unpackerProc = runtime.getNil();
-    if (packerArg != null) {
-      packerProc = packerArg.callMethod(ctx, "to_proc");
-    }
-    if (unpackerArg != null) {
-      if (unpackerArg instanceof RubyString || unpackerArg instanceof RubySymbol) {
-        unpackerProc = extModule.method(unpackerArg.callMethod(ctx, "to_sym"));
-      } else if (unpackerArg instanceof RubyProc || unpackerArg instanceof RubyMethod) {
-        unpackerProc = unpackerArg;
-      } else {
-        unpackerProc = unpackerArg.callMethod(ctx, "method", runtime.newSymbol("call"));
-      }
-    }
-
     boolean recursive = false;
     if (options != null) {
       IRubyObject recursiveExtensionArg = options.fastARef(runtime.newSymbol("recursive"));
@@ -151,7 +108,7 @@ public class Factory extends RubyObject {
       }
     }
 
-    extensionRegistry.put(extModule, (int) typeId, recursive, packerProc, packerArg, unpackerProc, unpackerArg);
+    extensionRegistry.put(extModule, (int) typeId, recursive, packerProc, unpackerProc);
 
     if (extModule == runtime.getSymbol() && !packerProc.isNil()) {
       hasSymbolExtType = true;

data/ext/java/org/msgpack/jruby/Packer.java

@@ -93,28 +93,11 @@ public class Packer extends RubyObject {
     return registry.toInternalPackerRegistry(ctx);
   }
 
-  @JRubyMethod(name = "register_type", required = 2, optional = 1)
-  public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args, final Block block) {
+  @JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
+  public IRubyObject registerType(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject proc) {
     testFrozen("MessagePack::Packer");
 
     Ruby runtime = ctx.runtime;
-    IRubyObject type = args[0];
-    IRubyObject mod = args[1];
-
-    IRubyObject arg;
-    IRubyObject proc;
-    if (args.length == 2) {
-      if (! block.isGiven()) {
-        throw runtime.newLocalJumpErrorNoBlock();
-      }
-      proc = block.getProcObject();
-      arg = proc;
-    } else if (args.length == 3) {
-      arg = args[2];
-      proc = arg.callMethod(ctx, "to_proc");
-    } else {
-      throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 2..3)", 2 + args.length));
-    }
 
     long typeId = ((RubyFixnum) type).getLongValue();
     if (typeId < -128 || typeId > 127) {
@@ -126,7 +109,7 @@ public class Packer extends RubyObject {
     }
     RubyModule extModule = (RubyModule) mod;
 
-    registry.put(extModule, (int) typeId, false, proc, arg, null, null);
+    registry.put(extModule, (int) typeId, false, proc, null);
 
     if (extModule == runtime.getSymbol() && !proc.isNil()) {
       encoder.hasSymbolExtType = true;

data/ext/java/org/msgpack/jruby/Unpacker.java

@@ -126,39 +126,23 @@ public class Unpacker extends RubyObject {
     return registry.toInternalUnpackerRegistry(ctx);
   }
 
-  @JRubyMethod(name = "register_type", required = 1, optional = 2)
-  public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args, final Block block) {
+  @JRubyMethod(name = "register_type_internal", required = 3, visibility = PRIVATE)
+  public IRubyObject registerTypeInternal(ThreadContext ctx, IRubyObject type, IRubyObject mod, IRubyObject proc) {
     testFrozen("MessagePack::Unpacker");
 
     Ruby runtime = ctx.runtime;
-    IRubyObject type = args[0];
-
-    RubyModule extModule;
-    IRubyObject arg;
-    IRubyObject proc;
-    if (args.length == 1) {
-      if (! block.isGiven()) {
-        throw runtime.newLocalJumpErrorNoBlock();
-      }
-      proc = RubyProc.newProc(runtime, block, block.type);
-      if (proc == null)
-        System.err.println("proc from Block is null");
-      arg = proc;
-      extModule = null;
-    } else if (args.length == 3) {
-      extModule = (RubyModule) args[1];
-      arg = args[2];
-      proc = extModule.method(arg);
-    } else {
-      throw runtime.newArgumentError(String.format("wrong number of arguments (%d for 1 or 3)", 2 + args.length));
-    }
 
     long typeId = ((RubyFixnum) type).getLongValue();
     if (typeId < -128 || typeId > 127) {
       throw runtime.newRangeError(String.format("integer %d too big to convert to `signed char'", typeId));
     }
 
-    registry.put(extModule, (int) typeId, false, null, null, proc, arg);
+    RubyModule extModule = null;
+    if (mod != runtime.getNil()) {
+      extModule = (RubyModule)mod;
+    }
+
+    registry.put(extModule, (int) typeId, false, null, proc);
     return runtime.getNil();
   }
 

data/ext/msgpack/buffer.c

@@ -251,12 +251,14 @@ bool _msgpack_buffer_read_all2(msgpack_buffer_t* b, char* buffer, size_t length)
 
 static inline msgpack_buffer_chunk_t* _msgpack_buffer_alloc_new_chunk(msgpack_buffer_t* b)
 {
-    msgpack_buffer_chunk_t* reuse = b->free_list;
-    if(reuse == NULL) {
-        return xmalloc(sizeof(msgpack_buffer_chunk_t));
+    msgpack_buffer_chunk_t* chunk = b->free_list;
+    if (chunk) {
+        b->free_list = b->free_list->next;
+    } else {
+        chunk = xmalloc(sizeof(msgpack_buffer_chunk_t));
     }
-    b->free_list = b->free_list->next;
-    return reuse;
+    memset(chunk, 0, sizeof(msgpack_buffer_chunk_t));
+    return chunk;
 }
 
 static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
@@ -298,7 +300,7 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
 static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE string)
 {
     VALUE mapped_string;
-    if(ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit && RTEST(rb_obj_frozen_p(string))) {
+    if(ENCODING_GET_INLINED(string) == msgpack_rb_encindex_ascii8bit && RB_OBJ_FROZEN_RAW(string)) {
         mapped_string = string;
     } else {
         mapped_string = rb_str_dup(string);
@@ -307,8 +309,9 @@ static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE s
 
     _msgpack_buffer_add_new_chunk(b);
 
-    char* data = RSTRING_PTR(mapped_string);
-    size_t length = RSTRING_LEN(mapped_string);
+    char* data;
+    size_t length;
+    RSTRING_GETMEM(mapped_string, data, length);
 
     b->tail.first = (char*) data;
     b->tail.last = (char*) data + length;
@@ -328,7 +331,7 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
 {
     if(b->io != Qnil) {
         msgpack_buffer_flush(b);
-        if (ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit) {
+        if (ENCODING_GET_INLINED(string) == msgpack_rb_encindex_ascii8bit) {
             rb_funcall(b->io, b->io_write_all_method, 1, string);
         } else {
             msgpack_buffer_append(b, RSTRING_PTR(string), RSTRING_LEN(string));

data/ext/msgpack/buffer.h

@@ -81,20 +81,6 @@ struct msgpack_buffer_chunk_t {
     bool rmem;
 };
 
-union msgpack_buffer_cast_block_t {
-    char buffer[8];
-    uint8_t u8;
-    uint16_t u16;
-    uint32_t u32;
-    uint64_t u64;
-    int8_t i8;
-    int16_t i16;
-    int32_t i32;
-    int64_t i64;
-    float f;
-    double d;
-};
-
 struct msgpack_buffer_t {
     char* read_buffer;
     char* tail_buffer_end;
@@ -107,8 +93,6 @@ struct msgpack_buffer_t {
     char* rmem_end;
     void** rmem_owner;
 
-    union msgpack_buffer_cast_block_t cast_block;
-
     VALUE io;
     VALUE io_buffer;
     ID io_write_all_method;
@@ -253,13 +237,14 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string);
 
 static inline size_t msgpack_buffer_append_string(msgpack_buffer_t* b, VALUE string)
 {
-    size_t length = RSTRING_LEN(string);
+    size_t length;
+    char *ptr;
+    RSTRING_GETMEM(string, ptr, length);
 
     if(length > b->write_reference_threshold) {
         _msgpack_buffer_append_long_string(b, string);
-
     } else {
-        msgpack_buffer_append(b, RSTRING_PTR(string), length);
+        msgpack_buffer_append(b, ptr, length);
     }
 
     return length;
@@ -268,7 +253,9 @@ static inline size_t msgpack_buffer_append_string(msgpack_buffer_t* b, VALUE str
 static inline size_t msgpack_buffer_append_string_reference(msgpack_buffer_t* b, VALUE string)
 {
     size_t length = RSTRING_LEN(string);
-    _msgpack_buffer_append_long_string(b, string);
+    if (length > 0) {
+        _msgpack_buffer_append_long_string(b, string);
+    }
     return length;
 }
 
@@ -381,14 +368,6 @@ static inline size_t msgpack_buffer_skip_nonblock(msgpack_buffer_t* b, size_t le
     return length;
 }
 
-static inline union msgpack_buffer_cast_block_t* msgpack_buffer_read_cast_block(msgpack_buffer_t* b, size_t n)
-{
-    if(!msgpack_buffer_read_all(b, b->cast_block.buffer, n)) {
-        return NULL;
-    }
-    return &b->cast_block;
-}
-
 size_t msgpack_buffer_read_to_string_nonblock(msgpack_buffer_t* b, VALUE string, size_t length);
 
 static inline size_t msgpack_buffer_read_to_string(msgpack_buffer_t* b, VALUE string, size_t length)
@@ -495,4 +474,131 @@ static inline VALUE msgpack_buffer_read_top_as_symbol(msgpack_buffer_t* b, size_
     return rb_str_intern(msgpack_buffer_read_top_as_string(b, length, true, utf8));
 }
 
+// Hash keys are likely to be repeated, and are frozen.
+// As such we can re-use them if we keep a cache of the ones we've seen so far,
+// and save much more expensive lookups into the global fstring table.
+// This cache implementation is deliberately simple, as we're optimizing for compactness,
+// to be able to fit easily embeded inside msgpack_unpacker_t.
+// As such, binary search into a sorted array gives a good tradeoff between compactness and
+// performance.
+#define MSGPACK_KEY_CACHE_CAPACITY 63
+
+typedef struct msgpack_key_cache_t msgpack_key_cache_t;
+struct msgpack_key_cache_t {
+    int length;
+    VALUE entries[MSGPACK_KEY_CACHE_CAPACITY];
+};
+
+static inline VALUE build_interned_string(const char *str, const long length)
+{
+# ifdef HAVE_RB_ENC_INTERNED_STR
+    return rb_enc_interned_str(str, length, rb_utf8_encoding());
+# else
+    VALUE rstring = rb_utf8_str_new(str, length);
+    return rb_funcall(rb_str_freeze(rstring), s_uminus, 0);
+# endif
+}
+
+static inline VALUE build_symbol(const char *str, const long length)
+{
+    return rb_str_intern(build_interned_string(str, length));
+}
+
+static void rvalue_cache_insert_at(msgpack_key_cache_t *cache, int index, VALUE rstring)
+{
+    MEMMOVE(&cache->entries[index + 1], &cache->entries[index], VALUE, cache->length - index);
+    cache->length++;
+    cache->entries[index] = rstring;
+}
+
+static inline int rstring_cache_cmp(const char *str, const long length, VALUE rstring)
+{
+    long rstring_length = RSTRING_LEN(rstring);
+    if (length == rstring_length) {
+        return memcmp(str, RSTRING_PTR(rstring), length);
+    } else {
+        return (int)(length - rstring_length);
+    }
+}
+
+static VALUE rstring_cache_fetch(msgpack_key_cache_t *cache, const char *str, const long length)
+{
+    int low = 0;
+    int high = cache->length - 1;
+    int mid = 0;
+    int last_cmp = 0;
+
+    while (low <= high) {
+        mid = (high + low) >> 1;
+        VALUE entry = cache->entries[mid];
+        last_cmp = rstring_cache_cmp(str, length, entry);
+
+        if (last_cmp == 0) {
+            return entry;
+        } else if (last_cmp > 0) {
+            low = mid + 1;
+        } else {
+            high = mid - 1;
+        }
+    }
+
+    VALUE rstring = build_interned_string(str, length);
+
+    if (cache->length < MSGPACK_KEY_CACHE_CAPACITY) {
+        if (last_cmp > 0) {
+            mid += 1;
+        }
+
+        rvalue_cache_insert_at(cache, mid, rstring);
+    }
+    return rstring;
+}
+
+static VALUE rsymbol_cache_fetch(msgpack_key_cache_t *cache, const char *str, const long length)
+{
+    int low = 0;
+    int high = cache->length - 1;
+    int mid = 0;
+    int last_cmp = 0;
+
+    while (low <= high) {
+        mid = (high + low) >> 1;
+        VALUE entry = cache->entries[mid];
+        last_cmp = rstring_cache_cmp(str, length, rb_sym2str(entry));
+
+        if (last_cmp == 0) {
+            return entry;
+        } else if (last_cmp > 0) {
+            low = mid + 1;
+        } else {
+            high = mid - 1;
+        }
+    }
+
+    VALUE rsymbol = build_symbol(str, length);
+
+    if (cache->length < MSGPACK_KEY_CACHE_CAPACITY) {
+        if (last_cmp > 0) {
+            mid += 1;
+        }
+
+        rvalue_cache_insert_at(cache, mid, rsymbol);
+    }
+    return rsymbol;
+}
+
+static inline VALUE msgpack_buffer_read_top_as_interned_symbol(msgpack_buffer_t* b, msgpack_key_cache_t *cache, size_t length)
+{
+    VALUE result = rsymbol_cache_fetch(cache, b->read_buffer, length);
+    _msgpack_buffer_consumed(b, length);
+    return result;
+}
+
+static inline VALUE msgpack_buffer_read_top_as_interned_string(msgpack_buffer_t* b, msgpack_key_cache_t *cache, size_t length)
+{
+    VALUE result = rstring_cache_fetch(cache, b->read_buffer, length);
+    _msgpack_buffer_consumed(b, length);
+    return result;
+}
+
 #endif

data/ext/msgpack/buffer_class.c

@@ -21,7 +21,8 @@
 #include "buffer.h"
 #include "buffer_class.h"
 
-VALUE cMessagePack_Buffer;
+VALUE cMessagePack_Buffer = Qnil;
+VALUE cMessagePack_HeldBuffer = Qnil;
 
 static ID s_read;
 static ID s_readpartial;
@@ -34,6 +35,73 @@ static VALUE sym_read_reference_threshold;
 static VALUE sym_write_reference_threshold;
 static VALUE sym_io_buffer_size;
 
+typedef struct msgpack_held_buffer_t msgpack_held_buffer_t;
+struct msgpack_held_buffer_t {
+    size_t size;
+    VALUE mapped_strings[];
+};
+
+static void HeldBuffer_mark(void *data)
+{
+    msgpack_held_buffer_t* held_buffer = (msgpack_held_buffer_t*)data;
+    for (size_t index = 0; index < held_buffer->size; index++) {
+        rb_gc_mark(held_buffer->mapped_strings[index]);
+    }
+}
+
+static size_t HeldBuffer_memsize(const void *data)
+{
+    const msgpack_held_buffer_t* held_buffer = (msgpack_held_buffer_t*)data;
+    return sizeof(size_t) + sizeof(VALUE) * held_buffer->size;
+}
+
+static const rb_data_type_t held_buffer_data_type = {
+    .wrap_struct_name = "msgpack:held_buffer",
+    .function = {
+        .dmark = HeldBuffer_mark,
+        .dfree = RUBY_TYPED_DEFAULT_FREE,
+        .dsize = HeldBuffer_memsize,
+    },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+VALUE MessagePack_Buffer_hold(msgpack_buffer_t* buffer)
+{
+    size_t mapped_strings_count = 0;
+    msgpack_buffer_chunk_t* c = buffer->head;
+    while (c != &buffer->tail) {
+        if (c->mapped_string != NO_MAPPED_STRING) {
+            mapped_strings_count++;
+        }
+        c = c->next;
+    }
+    if (c->mapped_string != NO_MAPPED_STRING) {
+        mapped_strings_count++;
+    }
+
+    if (mapped_strings_count == 0) {
+        return Qnil;
+    }
+
+    msgpack_held_buffer_t* held_buffer = xmalloc(sizeof(msgpack_held_buffer_t) + mapped_strings_count * sizeof(VALUE));
+
+    c = buffer->head;
+    mapped_strings_count = 0;
+    while (c != &buffer->tail) {
+        if (c->mapped_string != NO_MAPPED_STRING) {
+            held_buffer->mapped_strings[mapped_strings_count] = c->mapped_string;
+            mapped_strings_count++;
+        }
+        c = c->next;
+    }
+    if (c->mapped_string != NO_MAPPED_STRING) {
+        held_buffer->mapped_strings[mapped_strings_count] = c->mapped_string;
+        mapped_strings_count++;
+    }
+    held_buffer->size = mapped_strings_count;
+    return TypedData_Wrap_Struct(cMessagePack_HeldBuffer, &held_buffer_data_type, held_buffer);
+}
+
 
 #define CHECK_STRING_TYPE(value) \
     value = rb_check_string_type(value); \
@@ -520,6 +588,9 @@ void MessagePack_Buffer_module_init(VALUE mMessagePack)
 
     msgpack_buffer_static_init();
 
+    cMessagePack_HeldBuffer = rb_define_class_under(mMessagePack, "HeldBuffer", rb_cBasicObject);
+    rb_undef_alloc_func(cMessagePack_HeldBuffer);
+
     cMessagePack_Buffer = rb_define_class_under(mMessagePack, "Buffer", rb_cObject);
 
     rb_define_alloc_func(cMessagePack_Buffer, Buffer_alloc);

data/ext/msgpack/buffer_class.h

@@ -25,6 +25,7 @@ extern VALUE cMessagePack_Buffer;
 void MessagePack_Buffer_module_init(VALUE mMessagePack);
 
 VALUE MessagePack_Buffer_wrap(msgpack_buffer_t* b, VALUE owner);
+VALUE MessagePack_Buffer_hold(msgpack_buffer_t* b);
 
 void MessagePack_Buffer_set_options(msgpack_buffer_t* b, VALUE io, VALUE options);