msgpack 1.6.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3e9d5536bdf4c127f51b0c7837af3a3e076003f903aae018ba1332d1eecaa8e
-  data.tar.gz: c1f76401693a317b69bf37f8418e91f5a67dd77e74bd71668d321e0069b9ce7f
+  metadata.gz: 1414cbfba4f8503d851d6084ef42eb39cfe652d0cadedcf75523e5f472bf425b
+  data.tar.gz: a863d42ac8880a77afe84398b103f4b27d0782db70dd31aad4c785759bcfa20f
 SHA512:
-  metadata.gz: 18d189bec1b83c1e9252acaa82c3830728575a8222db0fe33fe3c1aa940734d4ee2c8a1991f2c38cef14ceccdde005ae4111df4884a8b43e4f0737e254ea719d
-  data.tar.gz: 52a5080b4ec7e62309e60ce4a29d8daaf1e39c2b03c87e310236afe053d31c8f878f304eb029b10d89393af0c7646873b0745f51cb6fa88e1f5e421aaa64e7c5
+  metadata.gz: 9ffd9d63c8354684cedfcd682beaf525823646aa9b0b6854175e81f1879eea22fcbf3fba560744210129001263ff80b2fc83ebc99a7405d9f03570921567b1d7
+  data.tar.gz: 4bd0b851caa3d39c05e32d6d90f9f52c376aae4f799247d3d9569db9bc273171c85973e6229de6b9c55e956e2f8c9790a9a4c23e8482944233ed499b421f30c6

data/ChangeLog

@@ -1,3 +1,11 @@
+2023-03-29 1.7.0:
+
+* Fix a possible double-free issue when GC triggers inside `_msgpack_rmem_alloc2`.
+* `Unpacker#feed` now always directly read in provided strings instead of copying content in its buffer.
+* `Unpacker#feed` is now an alias of `Unpacker#feed_reference`.
+* Implement `Factory::Pool#unpacker` and `Factory::Pool#packer` to allow for more precise serialization.
+* Require Ruby 2.5+.
+
 2023-03-03 1.6.1:
 
 * Undefine `#clone` and `#dup` on `MessagePack::Buffer`, `MessagePack::Packer` and `MessagePack::Unpacker`.

data/README.md

@@ -211,6 +211,33 @@ factory.register_type(
 factory.load(factory.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
 ```
 
+## Pooling
+
+Creating `Packer` and `Unpacker` objects is expensive. For best performance it is preferable to re-use these objects.
+
+`MessagePack::Factory#pool` makes that easier:
+
+```ruby
+factory = MessagePack::Factory.new
+factory.register_type(
+  0x01,
+  Point,
+  packer: ->(point, packer) {
+    packer.write(point.x)
+    packer.write(point.y)
+  },
+  unpacker: ->(unpacker) {
+    x = unpacker.read
+    y = unpacker.read
+    Point.new(x, y)
+  },
+  recursive: true,
+)
+pool = factory.pool(5) # The pool size should match the number of threads expected to use the factory concurrently.
+
+pool.load(pool.dump(Point.new(12, 34))) # => #<struct Point x=12, y=34>
+```
+
 ## Buffer API
 
 MessagePack for Ruby provides a buffer API so that you can read or write data by hand, not via Packer or Unpacker API.

data/ext/java/org/msgpack/jruby/Factory.java

@@ -92,7 +92,7 @@ public class Factory extends RubyObject {
     RubyHash options = null;
 
     if (isFrozen()) {
-        throw runtime.newRuntimeError("can't modify frozen Factory");
+        throw runtime.newFrozenError("MessagePack::Factory");
     }
 
     if (args.length == 2) {

data/ext/java/org/msgpack/jruby/Packer.java

@@ -96,6 +96,9 @@ public class Packer extends RubyObject {
   @JRubyMethod(name = "register_type", required = 2, optional = 1)
   public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args, final Block block) {
     Ruby runtime = ctx.runtime;
+    if (isFrozen()) {
+        throw runtime.newFrozenError("MessagePack::Packer");
+    }
     IRubyObject type = args[0];
     IRubyObject mod = args[1];
 

data/ext/java/org/msgpack/jruby/Unpacker.java

@@ -130,6 +130,9 @@ public class Unpacker extends RubyObject {
   @JRubyMethod(name = "register_type", required = 1, optional = 2)
   public IRubyObject registerType(ThreadContext ctx, IRubyObject[] args, final Block block) {
     Ruby runtime = ctx.runtime;
+    if (isFrozen()) {
+        throw runtime.newFrozenError("MessagePack::Unpacker");
+    }
     IRubyObject type = args[0];
 
     RubyModule extModule;

data/ext/msgpack/buffer.c

@@ -19,10 +19,6 @@
 #include "buffer.h"
 #include "rmem.h"
 
-#ifndef HAVE_RB_STR_REPLACE
-static ID s_replace;
-#endif
-
 int msgpack_rb_encindex_utf8;
 int msgpack_rb_encindex_usascii;
 int msgpack_rb_encindex_ascii8bit;
@@ -40,10 +36,6 @@ void msgpack_buffer_static_init(void)
     msgpack_rb_encindex_ascii8bit = rb_ascii8bit_encindex();
 
     msgpack_rmem_init(&s_rmem);
-
-#ifndef HAVE_RB_STR_REPLACE
-    s_replace = rb_intern("replace");
-#endif
 }
 
 void msgpack_buffer_static_destroy(void)
@@ -66,7 +58,11 @@ void msgpack_buffer_init(msgpack_buffer_t* b)
 static void _msgpack_buffer_chunk_destroy(msgpack_buffer_chunk_t* c)
 {
     if(c->mem != NULL) {
-        if(!msgpack_rmem_free(&s_rmem, c->mem)) {
+        if(c->rmem) {
+            if(!msgpack_rmem_free(&s_rmem, c->mem)) {
+                rb_bug("Failed to free an rmem pointer, memory leak?");
+            }
+        } else {
             xfree(c->mem);
         }
         /* no needs to update rmem_owner because chunks will not be
@@ -330,14 +326,12 @@ static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE s
 
 void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
 {
-    size_t length = RSTRING_LEN(string);
-
     if(b->io != Qnil) {
         msgpack_buffer_flush(b);
         if (ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit) {
             rb_funcall(b->io, b->io_write_all_method, 1, string);
         } else {
-            msgpack_buffer_append(b, RSTRING_PTR(string), length);
+            msgpack_buffer_append(b, RSTRING_PTR(string), RSTRING_LEN(string));
         }
     } else {
        _msgpack_buffer_append_reference(b, string);
@@ -349,6 +343,8 @@ static inline void* _msgpack_buffer_chunk_malloc(
         size_t required_size, size_t* allocated_size)
 {
     if(required_size <= MSGPACK_RMEM_PAGE_SIZE) {
+        c->rmem = true;
+
         if((size_t)(b->rmem_end - b->rmem_last) < required_size) {
             /* alloc new rmem page */
             *allocated_size = MSGPACK_RMEM_PAGE_SIZE;
@@ -379,6 +375,7 @@ static inline void* _msgpack_buffer_chunk_malloc(
     *allocated_size = required_size;
     void* mem = xmalloc(required_size);
     c->mem = mem;
+    c->rmem = false;
     return mem;
 }
 

data/ext/msgpack/buffer.h

@@ -78,6 +78,7 @@ struct msgpack_buffer_chunk_t {
     void* mem;
     msgpack_buffer_chunk_t* next;
     VALUE mapped_string;  /* RBString or NO_MAPPED_STRING */
+    bool rmem;
 };
 
 union msgpack_buffer_cast_block_t {
@@ -267,14 +268,7 @@ static inline size_t msgpack_buffer_append_string(msgpack_buffer_t* b, VALUE str
 static inline size_t msgpack_buffer_append_string_reference(msgpack_buffer_t* b, VALUE string)
 {
     size_t length = RSTRING_LEN(string);
-
-    if(length > MSGPACK_BUFFER_STRING_WRITE_REFERENCE_MINIMUM) {
-        _msgpack_buffer_append_long_string(b, string);
-
-    } else {
-        msgpack_buffer_append(b, RSTRING_PTR(string), length);
-    }
-
+    _msgpack_buffer_append_long_string(b, string);
     return length;
 }
 
@@ -479,7 +473,6 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
         result = rb_str_new(b->read_buffer, length);
     }
 
-#if STR_UMINUS_DEDUPE
     if (will_be_frozen) {
 #if STR_UMINUS_DEDUPE_FROZEN
         // Starting from MRI 2.8 it is preferable to freeze the string
@@ -491,7 +484,6 @@ static inline VALUE msgpack_buffer_read_top_as_string(msgpack_buffer_t* b, size_
         // frozen.
         result = rb_funcall(result, s_uminus, 0);
     }
-#endif // STR_UMINUS_DEDUPE
     _msgpack_buffer_consumed(b, length);
     return result;
 

data/ext/msgpack/extconf.rb

@@ -28,20 +28,6 @@ else
   $CFLAGS << ' -DHASH_ASET_DEDUPE=0 '
 end
 
-
-# checking if String#-@ (str_uminus) dedupes... ' (Ruby 2.5+)
-begin
-  a = -(%w(t e s t).join)
-  b = -(%w(t e s t).join)
-  if a.equal?(b)
-    $CFLAGS << ' -DSTR_UMINUS_DEDUPE=1 '
-  else
-    $CFLAGS += ' -DSTR_UMINUS_DEDUPE=0 '
-  end
-rescue NoMethodError
-  $CFLAGS << ' -DSTR_UMINUS_DEDUPE=0 '
-end
-
 # checking if String#-@ (str_uminus) directly interns frozen strings... ' (Ruby 3.0+)
 begin
   s = rand.to_s.freeze

data/ext/msgpack/factory_class.c

@@ -212,7 +212,7 @@ static VALUE Factory_register_type(int argc, VALUE* argv, VALUE self)
     VALUE packer_proc, unpacker_proc;
 
     if (OBJ_FROZEN(self)) {
-        rb_raise(rb_eRuntimeError, "can't modify frozen Factory");
+        rb_raise(rb_eFrozenError, "can't modify frozen MessagePack::Factory");
     }
 
     switch (argc) {

data/ext/msgpack/packer_class.c

@@ -349,6 +349,10 @@ static VALUE Packer_registered_types_internal(VALUE self)
 
 static VALUE Packer_register_type(int argc, VALUE* argv, VALUE self)
 {
+    if (OBJ_FROZEN(self)) {
+        rb_raise(rb_eFrozenError, "can't modify frozen MessagePack::Packer");
+    }
+
     msgpack_packer_t *pk = MessagePack_Packer_get(self);
 
     int ext_type;

data/ext/msgpack/rmem.c

@@ -65,11 +65,10 @@ void* _msgpack_rmem_alloc2(msgpack_rmem_t* pm)
     /* allocate new chunk */
     c = pm->array_last++;
 
-    /* move to head */
-    msgpack_rmem_chunk_t tmp = pm->head;
-    pm->head = *c;
-    *c = tmp;
+    /* move head to array */
+    *c = pm->head;
 
+    pm->head.pages = NULL; /* make sure we don't point to another chunk's pages in case xmalloc triggers GC */
     pm->head.mask = 0xffffffff & (~1);  /* "& (~1)" means first chunk is already allocated */
     pm->head.pages = xmalloc(MSGPACK_RMEM_PAGE_SIZE * 32);
 

data/ext/msgpack/unpacker.c

@@ -20,19 +20,17 @@
 #include "rmem.h"
 #include "extension_value_class.h"
 
-#if !defined(DISABLE_UNPACKER_STACK_RMEM) && \
-        MSGPACK_UNPACKER_STACK_CAPACITY * MSGPACK_UNPACKER_STACK_SIZE <= MSGPACK_RMEM_PAGE_SIZE
-#define UNPACKER_STACK_RMEM
-#endif
+_Static_assert(
+    sizeof(msgpack_unpacker_stack_entry_t) * MSGPACK_UNPACKER_STACK_CAPACITY <= MSGPACK_RMEM_PAGE_SIZE,
+    "msgpack_unpacker_stack_entry_t is too big to fit MSGPACK_UNPACKER_STACK_CAPACITY in MSGPACK_RMEM_PAGE_SIZE"
+);
 
 static int RAW_TYPE_STRING = 256;
 static int RAW_TYPE_BINARY = 257;
 
 static ID s_call;
 
-#ifdef UNPACKER_STACK_RMEM
 static msgpack_rmem_t s_stack_rmem;
-#endif
 
 #if !defined(HAVE_RB_HASH_NEW_CAPA)
 static inline VALUE rb_hash_new_capa(long capa)
@@ -43,18 +41,14 @@ static inline VALUE rb_hash_new_capa(long capa)
 
 void msgpack_unpacker_static_init(void)
 {
-#ifdef UNPACKER_STACK_RMEM
     msgpack_rmem_init(&s_stack_rmem);
-#endif
 
     s_call = rb_intern("call");
 }
 
 void msgpack_unpacker_static_destroy(void)
 {
-#ifdef UNPACKER_STACK_RMEM
     msgpack_rmem_destroy(&s_stack_rmem);
-#endif
 }
 
 #define HEAD_BYTE_REQUIRED 0xc1
@@ -62,13 +56,8 @@ void msgpack_unpacker_static_destroy(void)
 static inline msgpack_unpacker_stack_t* _msgpack_unpacker_new_stack(void) {
     msgpack_unpacker_stack_t *stack = ZALLOC(msgpack_unpacker_stack_t);
     stack->capacity = MSGPACK_UNPACKER_STACK_CAPACITY;
-#ifdef UNPACKER_STACK_RMEM
     stack->data = msgpack_rmem_alloc(&s_stack_rmem);
     /*memset(uk->stack, 0, MSGPACK_UNPACKER_STACK_CAPACITY);*/
-#else
-    /*uk->stack = calloc(MSGPACK_UNPACKER_STACK_CAPACITY, sizeof(msgpack_unpacker_stack_entry_t));*/
-    stack->data = xmalloc(MSGPACK_UNPACKER_STACK_CAPACITY * sizeof(msgpack_unpacker_stack_entry_t));
-#endif
     return stack;
 }
 
@@ -85,11 +74,9 @@ void _msgpack_unpacker_init(msgpack_unpacker_t* uk)
 }
 
 static inline void _msgpack_unpacker_free_stack(msgpack_unpacker_stack_t* stack) {
-    #ifdef UNPACKER_STACK_RMEM
-        msgpack_rmem_free(&s_stack_rmem, stack->data);
-    #else
-        xfree(stack->data);
-    #endif
+    if (!msgpack_rmem_free(&s_stack_rmem, stack->data)) {
+        rb_bug("Failed to free an rmem pointer, memory leak?");
+    }
     xfree(stack);
 }
 

data/ext/msgpack/unpacker.h

@@ -21,9 +21,7 @@
 #include "buffer.h"
 #include "unpacker_ext_registry.h"
 
-#ifndef MSGPACK_UNPACKER_STACK_CAPACITY
 #define MSGPACK_UNPACKER_STACK_CAPACITY 128
-#endif
 
 struct msgpack_unpacker_t;
 typedef struct msgpack_unpacker_t msgpack_unpacker_t;
@@ -49,8 +47,6 @@ struct msgpack_unpacker_stack_t {
     msgpack_unpacker_stack_t *parent;
 };
 
-#define MSGPACK_UNPACKER_STACK_SIZE (8+4+8+8)  /* assumes size_t <= 64bit, enum <= 32bit, VALUE <= 64bit */
-
 struct msgpack_unpacker_t {
     msgpack_buffer_t buffer;
     msgpack_unpacker_stack_t *stack;

data/ext/msgpack/unpacker_class.c

@@ -249,18 +249,6 @@ static VALUE Unpacker_read_map_header(VALUE self)
     return ULONG2NUM(size); // long at least 32 bits
 }
 
-
-static VALUE Unpacker_feed(VALUE self, VALUE data)
-{
-    msgpack_unpacker_t *uk = MessagePack_Unpacker_get(self);
-
-    StringValue(data);
-
-    msgpack_buffer_append_string(UNPACKER_BUFFER_(uk), data);
-
-    return self;
-}
-
 static VALUE Unpacker_feed_reference(VALUE self, VALUE data)
 {
     msgpack_unpacker_t *uk = MessagePack_Unpacker_get(self);
@@ -360,6 +348,10 @@ static VALUE Unpacker_registered_types_internal(VALUE self)
 
 static VALUE Unpacker_register_type(int argc, VALUE* argv, VALUE self)
 {
+    if (OBJ_FROZEN(self)) {
+        rb_raise(rb_eFrozenError, "can't modify frozen MessagePack::Unpacker");
+    }
+
     msgpack_unpacker_t *uk = MessagePack_Unpacker_get(self);
 
     int ext_type;
@@ -457,8 +449,8 @@ void MessagePack_Unpacker_module_init(VALUE mMessagePack)
     rb_define_method(cMessagePack_Unpacker, "skip_nil", Unpacker_skip_nil, 0);
     rb_define_method(cMessagePack_Unpacker, "read_array_header", Unpacker_read_array_header, 0);
     rb_define_method(cMessagePack_Unpacker, "read_map_header", Unpacker_read_map_header, 0);
-    rb_define_method(cMessagePack_Unpacker, "feed", Unpacker_feed, 1);
-    rb_define_method(cMessagePack_Unpacker, "feed_reference", Unpacker_feed_reference, 1);
+    rb_define_method(cMessagePack_Unpacker, "feed", Unpacker_feed_reference, 1);
+    rb_define_alias(cMessagePack_Unpacker, "feed_reference", "feed");
     rb_define_method(cMessagePack_Unpacker, "each", Unpacker_each, 0);
     rb_define_method(cMessagePack_Unpacker, "feed_each", Unpacker_feed_each, 1);
     rb_define_method(cMessagePack_Unpacker, "reset", Unpacker_reset, 0);

data/lib/msgpack/factory.rb

@@ -88,33 +88,34 @@ module MessagePack
 
     class Pool
       if RUBY_ENGINE == "ruby"
-        class AbstractPool
+        class MemberPool
           def initialize(size, &block)
             @size = size
             @new_member = block
             @members = []
           end
 
-          def checkout
-            @members.pop || @new_member.call
-          end
-
-          def checkin(member)
-            # If the pool is already full, we simply drop the extra member.
-            # This is because contrary to a connection pool, creating an extra instance
-            # is extremely unlikely to cause some kind of resource exhaustion.
-            #
-            # We could cycle the members (keep the newer one) but first It's more work and second
-            # the older member might have been created pre-fork, so it might be at least partially
-            # in shared memory.
-            if member && @members.size < @size
-              member.reset
-              @members << member
+          def with
+            member = @members.pop || @new_member.call
+            begin
+              yield member
+            ensure
+              # If the pool is already full, we simply drop the extra member.
+              # This is because contrary to a connection pool, creating an extra instance
+              # is extremely unlikely to cause some kind of resource exhaustion.
+              #
+              # We could cycle the members (keep the newer one) but first It's more work and second
+              # the older member might have been created pre-fork, so it might be at least partially
+              # in shared memory.
+              if member && @members.size < @size
+                member.reset
+                @members << member
+              end
             end
           end
         end
       else
-        class AbstractPool
+        class MemberPool
           def initialize(size, &block)
             @size = size
             @new_member = block
@@ -122,63 +123,50 @@ module MessagePack
             @mutex = Mutex.new
           end
 
-          def checkout
-            @mutex.synchronize { @members.pop } || @new_member.call
-          end
-
-          def checkin(member)
-            @mutex.synchronize do
-              if member && @members.size < @size
-                member.reset
-                @members << member
+          def with
+            member = @mutex.synchronize { @members.pop } || @new_member.call
+            begin
+              yield member
+            ensure
+              member.reset
+              @mutex.synchronize do
+                if member && @members.size < @size
+                  @members << member
+                end
               end
             end
           end
         end
       end
 
-      class PackerPool < AbstractPool
-        private
-
-        def reset(packer)
-          packer.clear
-        end
-      end
-
-      class UnpackerPool < AbstractPool
-        private
-
-        def reset(unpacker)
-          unpacker.reset
-        end
-      end
-
       def initialize(factory, size, options = nil)
         options = nil if !options || options.empty?
         @factory = factory
-        @packers = PackerPool.new(size) { factory.packer(options) }
-        @unpackers = UnpackerPool.new(size) { factory.unpacker(options) }
+        @packers = MemberPool.new(size) { factory.packer(options).freeze }
+        @unpackers = MemberPool.new(size) { factory.unpacker(options).freeze }
       end
 
       def load(data)
-        unpacker = @unpackers.checkout
-        begin
-          unpacker.feed_reference(data)
+        @unpackers.with do |unpacker|
+          unpacker.feed(data)
           unpacker.full_unpack
-        ensure
-          @unpackers.checkin(unpacker)
         end
       end
 
       def dump(object)
-        packer = @packers.checkout
-        begin
+        @packers.with do |packer|
           packer.write(object)
           packer.full_pack
-        ensure
-          @packers.checkin(packer)
         end
       end
+
+      def unpacker(&block)
+        @unpackers.with(&block)
+      end
+
+      def packer(&block)
+        @packers.with(&block)
+      end
     end
   end
 end

data/lib/msgpack/version.rb

@@ -1,5 +1,5 @@
 module MessagePack
-  VERSION = "1.6.1"
+  VERSION = "1.7.0"
   # Note for maintainers:
   #  Don't miss building/releasing the JRuby version (rake buld:java)
   #  See "How to build -java rubygems" in README for more details.

data/msgpack.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
     s.extensions = ["ext/msgpack/extconf.rb"]
   end
 
-  s.required_ruby_version = ">= 2.4"
+  s.required_ruby_version = ">= 2.5"
 
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'rake'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: msgpack
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-03 00:00:00.000000000 Z
+date: 2023-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -201,7 +201,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="