msgpack 1.7.2 → 1.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f5b1af6b3a51f5ccc6bcf67c94c1fc6193b02fe01b123e2cfb06a6df9607116
-  data.tar.gz: cc057f24e1ffa4cdc3e331499eb04de4c2383b0657dcf0baeba08300fd20862e
+  metadata.gz: ed5705d96741dbe59f61d59de79d133dfead8636cbf8c9db51b684c425a52e34
+  data.tar.gz: 6fa2a89acb1a2ae0db5068fa59b483d2292e0b9ad681f9c3c058cf164dd28385
 SHA512:
-  metadata.gz: 3eb06321a534ca9b16e321cc4a71458532578dafe7967314a662223b1fbf4aa93449c98177fa982aa532ce3732ddda4a6d497704df0e9c874da07f378c73595c
-  data.tar.gz: 8e540755e3db9e21d7dfa4354854e8b0486f5a1bbf82c3994c6095022205f7873153d364df9310d8072c481de38ca2b4c3e088e4221c3451ceb9438312489419
+  metadata.gz: 8edfe983aaee5230a1c1547e45894403c9c4435a65bbe4d88b4e9a800bb31467ad867526f4d54baf2cd6b7738cd9458104fc0caa772bba0dcbcb25c427002218
+  data.tar.gz: 3d282debd040b4f9b0f2a15c2c34068b64d01c696d1756f235c63078e54d5fcc2d1896f1b32321aa2c70ec67b8205eb608e841aff60eecd78a651e614016eac3

data/ChangeLog

@@ -1,3 +1,35 @@
+2026-06-10 1.8.3
+
+* Fix an integer overflow when parsing maps.
+
+2026-06-09 1.8.2
+
+* Fix `Buffer#clear` to properly reset memory chunks before adding them back to the pool.
+  This could have caused data to leak across buffers when using the MessagePack::Buffer API
+  directly. [CVE-PENDING].
+
+2026-05-28 1.8.1
+
+* Workaround rare compilation issue when `rb_hash_new_capa` isn't properly detected.
+* Never pre-allocate strings larger than the buffered size.
+
+2025-02-06 1.8.0
+
+* Numerous small optimizations.
+* Added `key_cache` option to `Unpacker`.
+
+2024-11-11 1.7.5
+
+* Rerelease 1.7.4 with fixed java package.
+
+2024-11-11 1.7.4
+
+* Fixed a potental memory leak when recursive unpacker raise.
+
+2024-10-03 1.7.3
+
+* Limit initial containers pre-allocation to `SHRT_MAX` (32k) entries.
+
 2023-07-18 1.7.2:
 
 * Fix a potential GC bug when packing data using recursive extensions and buffers containing over 512KkiB of data (See #341).

data/README.md

@@ -8,15 +8,24 @@ and typical short strings only require an extra byte in addition to the strings
 If you ever wished to use JSON for convenience (storing an image with metadata) but could
 not for technical reasons (binary data, size, speed...), MessagePack is a perfect replacement.
 
-    require 'msgpack'
-    msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
-    MessagePack.unpack(msg)   #=> [1,2,3]
+```ruby
+require 'msgpack'
+msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
+MessagePack.unpack(msg)   #=> [1,2,3]
+```
+
+Add msgpack to your Gemfile to install with Bundler:
+
+```ruby
+# Gemfile
+gem 'msgpack'
+```
 
-Use RubyGems to install:
+Or, use RubyGems to install:
 
     gem install msgpack
 
-or build msgpack-ruby and install:
+Or, build msgpack-ruby and install from a checked-out msgpack-ruby repository:
 
     bundle
     rake
@@ -27,11 +36,11 @@ or build msgpack-ruby and install:
 
 * Create REST API returing MessagePack using Rails + [RABL](https://github.com/nesquena/rabl)
 * Store objects efficiently serialized by msgpack on memcached or Redis
-  * In fact Redis supports msgpack in [EVAL-scripts](http://redis.io/commands/eval)
+  * In fact Redis supports msgpack in [EVAL-scripts](https://redis.io/docs/latest/commands/eval/)
 * Upload data in efficient format from mobile devices such as smartphones
   * MessagePack works on iPhone/iPad and Android. See also [Objective-C](https://github.com/msgpack/msgpack-objectivec) and [Java](https://github.com/msgpack/msgpack-java) implementations
 * Design a portable protocol to communicate with embedded devices
-  * Check also [Fluentd](http://fluentd.org/) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
+  * Check also [Fluentd](https://www.fluentd.org) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
 * Exchange objects between software components written in different languages
   * You'll need a flexible but efficient format so that components exchange objects while keeping compatibility
 
@@ -128,9 +137,9 @@ being serialized altogether by throwing an exception:
 
 ```ruby
 class Symbol
-    def to_msgpack_ext
-        raise "Serialization of symbols prohibited"
-    end
+  def to_msgpack_ext
+    raise "Serialization of symbols prohibited"
+  end
 end
 
 MessagePack::DefaultFactory.register_type(0x00, Symbol)
@@ -276,12 +285,12 @@ If this directory has Gemfile.lock (generated with MRI), remove it beforehand.
 
 ## Updating documents
 
-Online documents (http://ruby.msgpack.org) is generated from gh-pages branch.
-Following commands update documents in gh-pages branch:
+Online documentation (https://ruby.msgpack.org) is generated from the gh-pages branch.
+To update documents in gh-pages branch:
 
     bundle exec rake doc
     git checkout gh-pages
-    cp doc/* ./ -a
+    cp -a doc/* ./
 
 ## Copyright
 

data/ext/java/org/msgpack/jruby/Decoder.java

@@ -85,6 +85,14 @@ public class Decoder implements Iterator<IRubyObject> {
     feed(bytes, offset, length);
   }
 
+  private int getUnsignedInt() {
+    int size = buffer.getInt();
+    if (size < 0) {
+      throw runtime.newRaiseException(underflowErrorClass, "Size too large (limited to 2**31 for the Java version)");
+    }
+    return size;
+  }
+
   public void feed(byte[] bytes) {
     feed(bytes, 0, bytes.length);
   }
@@ -200,7 +208,7 @@ public class Decoder implements Iterator<IRubyObject> {
       } else if (b == ARY16) {
         return runtime.newFixnum(buffer.getShort() & 0xffff);
       } else if (b == ARY32) {
-        return runtime.newFixnum(buffer.getInt());
+        return runtime.newFixnum(getUnsignedInt());
       }
       throw runtime.newRaiseException(unexpectedTypeErrorClass, "unexpected type");
     } catch (RaiseException re) {
@@ -221,7 +229,7 @@ public class Decoder implements Iterator<IRubyObject> {
       } else if (b == MAP16) {
         return runtime.newFixnum(buffer.getShort() & 0xffff);
       } else if (b == MAP32) {
-        return runtime.newFixnum(buffer.getInt());
+        return runtime.newFixnum(getUnsignedInt());
       }
       throw runtime.newRaiseException(unexpectedTypeErrorClass, "unexpected type");
     } catch (RaiseException re) {
@@ -258,10 +266,10 @@ public class Decoder implements Iterator<IRubyObject> {
         case TRUE:     return runtime.getTrue();
         case BIN8:     return consumeString(buffer.get() & 0xff, binaryEncoding);
         case BIN16:    return consumeString(buffer.getShort() & 0xffff, binaryEncoding);
-        case BIN32:    return consumeString(buffer.getInt(), binaryEncoding);
+        case BIN32:    return consumeString(getUnsignedInt(), binaryEncoding);
         case VAREXT8:  return consumeExtension(buffer.get() & 0xff);
         case VAREXT16: return consumeExtension(buffer.getShort() & 0xffff);
-        case VAREXT32: return consumeExtension(buffer.getInt());
+        case VAREXT32: return consumeExtension(getUnsignedInt());
         case FLOAT32:  return runtime.newFloat(buffer.getFloat());
         case FLOAT64:  return runtime.newFloat(buffer.getDouble());
         case UINT8:    return runtime.newFixnum(buffer.get() & 0xffL);
@@ -283,11 +291,11 @@ public class Decoder implements Iterator<IRubyObject> {
         case FIXEXT16: return consumeExtension(16);
         case STR8:     return consumeString(buffer.get() & 0xff, utf8Encoding);
         case STR16:    return consumeString(buffer.getShort() & 0xffff, utf8Encoding);
-        case STR32:    return consumeString(buffer.getInt(), utf8Encoding);
+        case STR32:    return consumeString(getUnsignedInt(), utf8Encoding);
         case ARY16:    return consumeArray(buffer.getShort() & 0xffff);
-        case ARY32:    return consumeArray(buffer.getInt());
+        case ARY32:    return consumeArray(getUnsignedInt());
         case MAP16:    return consumeHash(buffer.getShort() & 0xffff);
-        case MAP32:    return consumeHash(buffer.getInt());
+        case MAP32:    return consumeHash(getUnsignedInt());
         default: break outer;
         }
       case 0xe:

data/ext/msgpack/buffer.c

@@ -134,6 +134,9 @@ bool _msgpack_buffer_shift_chunk(msgpack_buffer_t* b)
          * because head should be always available */
         b->tail_buffer_end = NULL;
         b->read_buffer = NULL;
+        b->rmem_end = NULL;
+        b->rmem_last = NULL;
+        b->rmem_owner = NULL;
         return false;
     }
 
@@ -300,7 +303,7 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
 static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE string)
 {
     VALUE mapped_string;
-    if(ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit && RTEST(rb_obj_frozen_p(string))) {
+    if(ENCODING_GET_INLINED(string) == msgpack_rb_encindex_ascii8bit && RB_OBJ_FROZEN_RAW(string)) {
         mapped_string = string;
     } else {
         mapped_string = rb_str_dup(string);
@@ -309,8 +312,9 @@ static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE s
 
     _msgpack_buffer_add_new_chunk(b);
 
-    char* data = RSTRING_PTR(mapped_string);
-    size_t length = RSTRING_LEN(mapped_string);
+    char* data;
+    size_t length;
+    RSTRING_GETMEM(mapped_string, data, length);
 
     b->tail.first = (char*) data;
     b->tail.last = (char*) data + length;
@@ -330,7 +334,7 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
 {
     if(b->io != Qnil) {
         msgpack_buffer_flush(b);
-        if (ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit) {
+        if (ENCODING_GET_INLINED(string) == msgpack_rb_encindex_ascii8bit) {
             rb_funcall(b->io, b->io_write_all_method, 1, string);
         } else {
             msgpack_buffer_append(b, RSTRING_PTR(string), RSTRING_LEN(string));

data/ext/msgpack/buffer.h

@@ -81,20 +81,6 @@ struct msgpack_buffer_chunk_t {
     bool rmem;
 };
 
-union msgpack_buffer_cast_block_t {
-    char buffer[8];
-    uint8_t u8;
-    uint16_t u16;
-    uint32_t u32;
-    uint64_t u64;
-    int8_t i8;
-    int16_t i16;
-    int32_t i32;
-    int64_t i64;
-    float f;
-    double d;
-};
-
 struct msgpack_buffer_t {
     char* read_buffer;
     char* tail_buffer_end;
@@ -107,8 +93,6 @@ struct msgpack_buffer_t {
     char* rmem_end;
     void** rmem_owner;
 
-    union msgpack_buffer_cast_block_t cast_block;
-
     VALUE io;
     VALUE io_buffer;
     ID io_write_all_method;
@@ -253,13 +237,14 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string);
 
 static inline size_t msgpack_buffer_append_string(msgpack_buffer_t* b, VALUE string)
 {
-    size_t length = RSTRING_LEN(string);
+    size_t length;
+    char *ptr;
+    RSTRING_GETMEM(string, ptr, length);
 
     if(length > b->write_reference_threshold) {
         _msgpack_buffer_append_long_string(b, string);
-
     } else {
-        msgpack_buffer_append(b, RSTRING_PTR(string), length);
+        msgpack_buffer_append(b, ptr, length);
     }
 
     return length;
@@ -383,14 +368,6 @@ static inline size_t msgpack_buffer_skip_nonblock(msgpack_buffer_t* b, size_t le
     return length;
 }
 
-static inline union msgpack_buffer_cast_block_t* msgpack_buffer_read_cast_block(msgpack_buffer_t* b, size_t n)
-{
-    if(!msgpack_buffer_read_all(b, b->cast_block.buffer, n)) {
-        return NULL;
-    }
-    return &b->cast_block;
-}
-
 size_t msgpack_buffer_read_to_string_nonblock(msgpack_buffer_t* b, VALUE string, size_t length);
 
 static inline size_t msgpack_buffer_read_to_string(msgpack_buffer_t* b, VALUE string, size_t length)
@@ -497,4 +474,131 @@ static inline VALUE msgpack_buffer_read_top_as_symbol(msgpack_buffer_t* b, size_
     return rb_str_intern(msgpack_buffer_read_top_as_string(b, length, true, utf8));
 }
 
+// Hash keys are likely to be repeated, and are frozen.
+// As such we can re-use them if we keep a cache of the ones we've seen so far,
+// and save much more expensive lookups into the global fstring table.
+// This cache implementation is deliberately simple, as we're optimizing for compactness,
+// to be able to fit easily embeded inside msgpack_unpacker_t.
+// As such, binary search into a sorted array gives a good tradeoff between compactness and
+// performance.
+#define MSGPACK_KEY_CACHE_CAPACITY 63
+
+typedef struct msgpack_key_cache_t msgpack_key_cache_t;
+struct msgpack_key_cache_t {
+    int length;
+    VALUE entries[MSGPACK_KEY_CACHE_CAPACITY];
+};
+
+static inline VALUE build_interned_string(const char *str, const long length)
+{
+# ifdef HAVE_RB_ENC_INTERNED_STR
+    return rb_enc_interned_str(str, length, rb_utf8_encoding());
+# else
+    VALUE rstring = rb_utf8_str_new(str, length);
+    return rb_funcall(rb_str_freeze(rstring), s_uminus, 0);
+# endif
+}
+
+static inline VALUE build_symbol(const char *str, const long length)
+{
+    return rb_str_intern(build_interned_string(str, length));
+}
+
+static void rvalue_cache_insert_at(msgpack_key_cache_t *cache, int index, VALUE rstring)
+{
+    MEMMOVE(&cache->entries[index + 1], &cache->entries[index], VALUE, cache->length - index);
+    cache->length++;
+    cache->entries[index] = rstring;
+}
+
+static inline int rstring_cache_cmp(const char *str, const long length, VALUE rstring)
+{
+    long rstring_length = RSTRING_LEN(rstring);
+    if (length == rstring_length) {
+        return memcmp(str, RSTRING_PTR(rstring), length);
+    } else {
+        return (int)(length - rstring_length);
+    }
+}
+
+static VALUE rstring_cache_fetch(msgpack_key_cache_t *cache, const char *str, const long length)
+{
+    int low = 0;
+    int high = cache->length - 1;
+    int mid = 0;
+    int last_cmp = 0;
+
+    while (low <= high) {
+        mid = (high + low) >> 1;
+        VALUE entry = cache->entries[mid];
+        last_cmp = rstring_cache_cmp(str, length, entry);
+
+        if (last_cmp == 0) {
+            return entry;
+        } else if (last_cmp > 0) {
+            low = mid + 1;
+        } else {
+            high = mid - 1;
+        }
+    }
+
+    VALUE rstring = build_interned_string(str, length);
+
+    if (cache->length < MSGPACK_KEY_CACHE_CAPACITY) {
+        if (last_cmp > 0) {
+            mid += 1;
+        }
+
+        rvalue_cache_insert_at(cache, mid, rstring);
+    }
+    return rstring;
+}
+
+static VALUE rsymbol_cache_fetch(msgpack_key_cache_t *cache, const char *str, const long length)
+{
+    int low = 0;
+    int high = cache->length - 1;
+    int mid = 0;
+    int last_cmp = 0;
+
+    while (low <= high) {
+        mid = (high + low) >> 1;
+        VALUE entry = cache->entries[mid];
+        last_cmp = rstring_cache_cmp(str, length, rb_sym2str(entry));
+
+        if (last_cmp == 0) {
+            return entry;
+        } else if (last_cmp > 0) {
+            low = mid + 1;
+        } else {
+            high = mid - 1;
+        }
+    }
+
+    VALUE rsymbol = build_symbol(str, length);
+
+    if (cache->length < MSGPACK_KEY_CACHE_CAPACITY) {
+        if (last_cmp > 0) {
+            mid += 1;
+        }
+
+        rvalue_cache_insert_at(cache, mid, rsymbol);
+    }
+    return rsymbol;
+}
+
+static inline VALUE msgpack_buffer_read_top_as_interned_symbol(msgpack_buffer_t* b, msgpack_key_cache_t *cache, size_t length)
+{
+    VALUE result = rsymbol_cache_fetch(cache, b->read_buffer, length);
+    _msgpack_buffer_consumed(b, length);
+    return result;
+}
+
+static inline VALUE msgpack_buffer_read_top_as_interned_string(msgpack_buffer_t* b, msgpack_key_cache_t *cache, size_t length)
+{
+    VALUE result = rstring_cache_fetch(cache, b->read_buffer, length);
+    _msgpack_buffer_consumed(b, length);
+    return result;
+}
+
 #endif

data/ext/msgpack/extconf.rb

@@ -3,17 +3,19 @@ require 'mkmf'
 have_func("rb_enc_interned_str", "ruby.h") # Ruby 3.0+
 have_func("rb_hash_new_capa", "ruby.h") # Ruby 3.2+
 have_func("rb_proc_call_with_block", "ruby.h") # CRuby (TruffleRuby doesn't have it)
+have_func("rb_gc_mark_locations", "ruby.h") # Missing on TruffleRuby
 
 append_cflags([
   "-fvisibility=hidden",
   "-I..",
   "-Wall",
-  "-O3",
   "-std=gnu99"
 ])
-append_cflags(RbConfig::CONFIG["debugflags"]) if RbConfig::CONFIG["debugflags"]
 
-append_cflags("-DRUBY_DEBUG=1") if ENV["MSGPACK_DEBUG"]
+if ENV["MSGPACK_DEBUG"]
+  append_cflags(RbConfig::CONFIG["debugflags"]) if RbConfig::CONFIG["debugflags"]
+  append_cflags("-DRUBY_DEBUG=1")
+end
 
 if RUBY_VERSION.start_with?('3.0.') && RUBY_VERSION <= '3.0.5'
   # https://bugs.ruby-lang.org/issues/18772

data/ext/msgpack/packer.h

@@ -25,21 +25,26 @@
 #define MSGPACK_PACKER_IO_FLUSH_THRESHOLD_TO_WRITE_STRING_BODY (1024)
 #endif
 
+#ifndef UNREACHABLE_RETURN
+// Ruby 2.5
+#define UNREACHABLE_RETURN() return
+#endif
+
 struct msgpack_packer_t;
 typedef struct msgpack_packer_t msgpack_packer_t;
 
 struct msgpack_packer_t {
     msgpack_buffer_t buffer;
 
-    bool compatibility_mode;
-    bool has_bigint_ext_type;
-    bool has_symbol_ext_type;
-
     ID to_msgpack_method;
     VALUE to_msgpack_arg;
 
     VALUE buffer_ref;
 
+    bool compatibility_mode;
+    bool has_bigint_ext_type;
+    bool has_symbol_ext_type;
+
     /* options */
     bool comaptibility_mode;
     msgpack_packer_ext_registry_t ext_registry;
@@ -404,27 +409,33 @@ static inline bool msgpack_packer_is_utf8_compat_string(VALUE v, int encindex)
 {
     return encindex == msgpack_rb_encindex_utf8
         || encindex == msgpack_rb_encindex_usascii
-        || (rb_enc_asciicompat(rb_enc_from_index(encindex)) && ENC_CODERANGE_ASCIIONLY(v));
+        || ENC_CODERANGE_ASCIIONLY(v);
 }
 
 static inline void msgpack_packer_write_string_value(msgpack_packer_t* pk, VALUE v)
 {
-    /* actual return type of RSTRING_LEN is long */
-    unsigned long len = RSTRING_LEN(v);
-    if(len > 0xffffffffUL) {
-        // TODO rb_eArgError?
-        rb_raise(rb_eArgError, "size of string is too long to pack: %lu bytes should be <= %lu", len, 0xffffffffUL);
+    long len = RSTRING_LEN(v);
+
+    if(RB_UNLIKELY(len > 0xffffffffL)) {
+        rb_raise(rb_eArgError, "size of string is too long to pack: %lu bytes should be <= %ld", len, 0xffffffffL);
+        UNREACHABLE_RETURN();
+    }
+
+    if (RB_UNLIKELY(pk->compatibility_mode)) {
+        msgpack_packer_write_raw_header(pk, (unsigned int)len);
+        msgpack_buffer_append_string(PACKER_BUFFER_(pk), v);
+        return;
     }
 
-    int encindex = ENCODING_GET(v);
-    if(msgpack_packer_is_binary(v, encindex) && !pk->compatibility_mode) {
+    int encindex = ENCODING_GET_INLINED(v);
+    if(msgpack_packer_is_binary(v, encindex)) {
         /* write ASCII-8BIT string using Binary type */
         msgpack_packer_write_bin_header(pk, (unsigned int)len);
         msgpack_buffer_append_string(PACKER_BUFFER_(pk), v);
     } else {
         /* write UTF-8, US-ASCII, or 7bit-safe ascii-compatible string using String type directly */
         /* in compatibility mode, packer packs String values as is */
-        if(!pk->compatibility_mode && !msgpack_packer_is_utf8_compat_string(v, encindex)) {
+        if(RB_UNLIKELY(!msgpack_packer_is_utf8_compat_string(v, encindex))) {
             /* transcode other strings to UTF-8 and write using String type */
             VALUE enc = rb_enc_from_encoding(rb_utf8_encoding()); /* rb_enc_from_encoding_index is not extern */
             v = rb_str_encode(v, enc, 0, Qnil);
@@ -453,11 +464,7 @@ static inline void msgpack_packer_write_symbol_value(msgpack_packer_t* pk, VALUE
 
 static inline void msgpack_packer_write_fixnum_value(msgpack_packer_t* pk, VALUE v)
 {
-#ifdef JRUBY
-    msgpack_packer_write_long(pk, FIXNUM_P(v) ? FIX2LONG(v) : rb_num2ll(v));
-#else
     msgpack_packer_write_long(pk, FIX2LONG(v));
-#endif
 }
 
 static inline void msgpack_packer_write_bignum_value(msgpack_packer_t* pk, VALUE v)