google-protobuf 3.15.3-x86-linux → 3.16.0.rc.1-x86-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: baa9d45f0c885758e32b6d221977ef378567b0296dd0aa15b5b2e28801a662bd
-  data.tar.gz: 33c18b9fb8e2de390a19cec01d5f73bbc2b2089dd1bf3a96ead061822007437d
+  metadata.gz: 57dfaa408109ee7a9c22b44623711fb8c636f61734770091f0cfa5d81e157d6b
+  data.tar.gz: a948993f9f83cb9fd75b9090f795dbd0a3119e9a1c770fe4121f7a6ba13b4f8a
 SHA512:
-  metadata.gz: 6fee0801f9c427747d0511023a81fa2d6199308178b08e42c4e75af8277263311b4a6057e0fabb4befe97eebae67e04d6e3694cf9ff0e94a2670128d4b3239df
-  data.tar.gz: 436d37545240010cb2bcc0d5429b2727ba81c161cb95486a5d6a2a40ef09675eee6eb59eb512c4a79fc1fd30ea5bc7f1fc82deb6b9f1cb139e023778fcc15398
+  metadata.gz: d2394db83f6f2d7243eab2dd788cd77fe156d849966e6c4551c61f90ce4a97acf8c727b0efdff6ad7e3f6b19f8d7fed575ff918b9921c706c553c42fe73c074a
+  data.tar.gz: 9ac3ad884829f1bc2ca83aad0da07f45f1c34d7646c1580b3f0c72ad60f7ab6fd7e93010cb15ab4a64fe70a8080568203faedb7c3b4715cb71c81d85649e739b

data/ext/google/protobuf_c/convert.c

@@ -315,7 +315,7 @@ bool Msgval_IsEqual(upb_msgval val1, upb_msgval val2, TypeInfo type_info) {
       return memcmp(&val1, &val2, 8) == 0;
     case UPB_TYPE_STRING:
     case UPB_TYPE_BYTES:
-      return val1.str_val.size != val2.str_val.size ||
+      return val1.str_val.size == val2.str_val.size &&
              memcmp(val1.str_val.data, val2.str_val.data,
                     val1.str_val.size) == 0;
     case UPB_TYPE_MESSAGE:

data/ext/google/protobuf_c/message.c

@@ -697,16 +697,13 @@ bool Message_Equal(const upb_msg *m1, const upb_msg *m2, const upb_msgdef *m) {
  * field is of a primitive type).
  */
 static VALUE Message_eq(VALUE _self, VALUE _other) {
-  if (TYPE(_self) != TYPE(_other)) {
-    return Qfalse;
-  }
+  if (CLASS_OF(_self) != CLASS_OF(_other)) return Qfalse;
 
   Message* self = ruby_to_Message(_self);
   Message* other = ruby_to_Message(_other);
+  assert(self->msgdef == other->msgdef);
 
-  return Message_Equal(self->msg, other->msg, self->msgdef)
-             ? Qtrue
-             : Qfalse;
+  return Message_Equal(self->msg, other->msg, self->msgdef) ? Qtrue : Qfalse;
 }
 
 uint64_t Message_Hash(const upb_msg* msg, const upb_msgdef* m, uint64_t seed) {
@@ -1250,7 +1247,9 @@ upb_msg* Message_deep_copy(const upb_msg* msg, const upb_msgdef* m,
 
 const upb_msg* Message_GetUpbMessage(VALUE value, const upb_msgdef* m,
                                      const char* name, upb_arena* arena) {
-  if (value == Qnil) return NULL;
+  if (value == Qnil) {
+    rb_raise(cTypeError, "nil message not allowed here.");
+  }
 
   VALUE klass = CLASS_OF(value);
   VALUE desc_rb = rb_ivar_get(klass, descriptor_instancevar_interned);

data/ext/google/protobuf_c/protobuf.c

@@ -37,7 +37,7 @@
 #include "message.h"
 #include "repeated_field.h"
 
-VALUE cError;
+VALUE cParseError;
 VALUE cTypeError;
 
 const upb_fielddef* map_field_key(const upb_fielddef* field) {
@@ -251,14 +251,80 @@ void Arena_register(VALUE module) {
 // The object is used only for its identity; it does not contain any data.
 VALUE secondary_map = Qnil;
 
+// Mutations to the map are under a mutex, because SeconaryMap_MaybeGC()
+// iterates over the map which cannot happen in parallel with insertions, or
+// Ruby will throw:
+//   can't add a new key into hash during iteration (RuntimeError)
+VALUE secondary_map_mutex = Qnil;
+
+// Lambda that will GC entries from the secondary map that are no longer present
+// in the primary map.
+VALUE gc_secondary_map_lambda = Qnil;
+ID length;
+
+extern VALUE weak_obj_cache;
+
 static void SecondaryMap_Init() {
   rb_gc_register_address(&secondary_map);
+  rb_gc_register_address(&gc_secondary_map_lambda);
+  rb_gc_register_address(&secondary_map_mutex);
   secondary_map = rb_hash_new();
+  gc_secondary_map_lambda = rb_eval_string(
+      "->(secondary, weak) {\n"
+      "  secondary.delete_if { |k, v| !weak.key?(v) }\n"
+      "}\n");
+  secondary_map_mutex = rb_mutex_new();
+  length = rb_intern("length");
 }
 
-static VALUE SecondaryMap_Get(VALUE key) {
+// The secondary map is a regular Hash, and will never shrink on its own.
+// The main object cache is a WeakMap that will automatically remove entries
+// when the target object is no longer reachable, but unless we manually
+// remove the corresponding entries from the secondary map, it will grow
+// without bound.
+//
+// To avoid this unbounded growth we periodically remove entries from the
+// secondary map that are no longer present in the WeakMap. The logic of
+// how often to perform this GC is an artbirary tuning parameter that
+// represents a straightforward CPU/memory tradeoff.
+//
+// Requires: secondary_map_mutex is held.
+static void SecondaryMap_MaybeGC() {
+  PBRUBY_ASSERT(rb_mutex_locked_p(secondary_map_mutex) == Qtrue);
+  size_t weak_len = NUM2ULL(rb_funcall(weak_obj_cache, length, 0));
+  size_t secondary_len = RHASH_SIZE(secondary_map);
+  if (secondary_len < weak_len) {
+    // Logically this case should not be possible: a valid entry cannot exist in
+    // the weak table unless there is a corresponding entry in the secondary
+    // table. It should *always* be the case that secondary_len >= weak_len.
+    //
+    // However ObjectSpace::WeakMap#length (and therefore weak_len) is
+    // unreliable: it overreports its true length by including non-live objects.
+    // However these non-live objects are not yielded in iteration, so we may
+    // have previously deleted them from the secondary map in a previous
+    // invocation of SecondaryMap_MaybeGC().
+    //
+    // In this case, we can't measure any waste, so we just return.
+    return;
+  }
+  size_t waste = secondary_len - weak_len;
+  // GC if we could remove at least 2000 entries or 20% of the table size
+  // (whichever is greater).  Since the cost of the GC pass is O(N), we
+  // want to make sure that we condition this on overall table size, to
+  // avoid O(N^2) CPU costs.
+  size_t threshold = PBRUBY_MAX(secondary_len * 0.2, 2000);
+  if (waste > threshold) {
+    rb_funcall(gc_secondary_map_lambda, rb_intern("call"), 2,
+               secondary_map, weak_obj_cache);
+  }
+}
+
+// Requires: secondary_map_mutex is held by this thread iff create == true.
+static VALUE SecondaryMap_Get(VALUE key, bool create) {
+  PBRUBY_ASSERT(!create || rb_mutex_locked_p(secondary_map_mutex) == Qtrue);
   VALUE ret = rb_hash_lookup(secondary_map, key);
-  if (ret == Qnil) {
+  if (ret == Qnil && create) {
+    SecondaryMap_MaybeGC();
     ret = rb_eval_string("Object.new");
     rb_hash_aset(secondary_map, key, ret);
   }
@@ -267,14 +333,15 @@ static VALUE SecondaryMap_Get(VALUE key) {
 
 #endif
 
-static VALUE ObjectCache_GetKey(const void* key) {
+// Requires: secondary_map_mutex is held by this thread iff create == true.
+static VALUE ObjectCache_GetKey(const void* key, bool create) {
   char buf[sizeof(key)];
   memcpy(&buf, &key, sizeof(key));
   intptr_t key_int = (intptr_t)key;
   PBRUBY_ASSERT((key_int & 3) == 0);
   VALUE ret = LL2NUM(key_int >> 2);
 #if USE_SECONDARY_MAP
-  ret = SecondaryMap_Get(ret);
+  ret = SecondaryMap_Get(ret, create);
 #endif
   return ret;
 }
@@ -298,14 +365,20 @@ static void ObjectCache_Init() {
 
 void ObjectCache_Add(const void* key, VALUE val) {
   PBRUBY_ASSERT(ObjectCache_Get(key) == Qnil);
-  VALUE key_rb = ObjectCache_GetKey(key);
+#if USE_SECONDARY_MAP
+  rb_mutex_lock(secondary_map_mutex);
+#endif
+  VALUE key_rb = ObjectCache_GetKey(key, true);
   rb_funcall(weak_obj_cache, item_set, 2, key_rb, val);
+#if USE_SECONDARY_MAP
+  rb_mutex_unlock(secondary_map_mutex);
+#endif
   PBRUBY_ASSERT(ObjectCache_Get(key) == val);
 }
 
 // Returns the cached object for this key, if any. Otherwise returns Qnil.
 VALUE ObjectCache_Get(const void* key) {
-  VALUE key_rb = ObjectCache_GetKey(key);
+  VALUE key_rb = ObjectCache_GetKey(key, false);
   return rb_funcall(weak_obj_cache, item_get, 1, key_rb);
 }
 
@@ -368,8 +441,10 @@ void Init_protobuf_c() {
   Map_register(protobuf);
   Message_register(protobuf);
 
-  cError = rb_const_get(protobuf, rb_intern("Error"));
+  cParseError = rb_const_get(protobuf, rb_intern("ParseError"));
+  rb_gc_register_mark_object(cParseError);
   cTypeError = rb_const_get(protobuf, rb_intern("TypeError"));
+  rb_gc_register_mark_object(cTypeError);
 
   rb_define_singleton_method(protobuf, "discard_unknown",
                              Google_Protobuf_discard_unknown, 1);

data/ext/google/protobuf_c/protobuf.h

@@ -106,6 +106,8 @@ extern VALUE cTypeError;
 #define PBRUBY_ASSERT(expr) assert(expr)
 #endif
 
+#define PBRUBY_MAX(x, y) (((x) > (y)) ? (x) : (y))
+
 #define UPB_UNUSED(var) (void)var
 
 #endif  // __GOOGLE_PROTOBUF_RUBY_PROTOBUF_H__

data/ext/google/protobuf_c/ruby-upb.c

@@ -6663,10 +6663,9 @@ void upb_array_set(upb_array *arr, size_t i, upb_msgval val) {
 }
 
 bool upb_array_append(upb_array *arr, upb_msgval val, upb_arena *arena) {
-  if (!_upb_array_realloc(arr, arr->len + 1, arena)) {
+  if (!upb_array_resize(arr, arr->len + 1, arena)) {
     return false;
   }
-  arr->len++;
   upb_array_set(arr, arr->len - 1, val);
   return true;
 }

data/tests/basic.rb

@@ -52,10 +52,15 @@ module BasicTest
 
       outer = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("Outer").msgclass
 
-      outer_proto = outer.new(
+      outer.new(
           inners: []
-      )
-      outer_proto['inners'].to_s
+      )['inners'].to_s
+
+      assert_raise Google::Protobuf::TypeError do
+        outer.new(
+            inners: [nil]
+        ).to_s
+      end
     end
 
     def test_has_field

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google-protobuf
 version: !ruby/object:Gem::Version
-  version: 3.15.3
+  version: 3.16.0.rc.1
 platform: x86-linux
 authors:
 - Protobuf Authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-25 00:00:00.000000000 Z
+date: 2021-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler-dock
@@ -128,7 +128,7 @@ homepage: https://developers.google.com/protocol-buffers
 licenses:
 - BSD-3-Clause
 metadata:
-  source_code_uri: https://github.com/protocolbuffers/protobuf/tree/v3.15.3/ruby
+  source_code_uri: https://github.com/protocolbuffers/protobuf/tree/v3.16.0-rc1/ruby
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -143,9 +143,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 3.1.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.2.3
 signing_key: