google-protobuf 3.15.6 → 3.17.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6f2d2b024b64eb8c142ff7eaea74a58431a8a12f936ab7152ab5c8721efd121
-  data.tar.gz: 4ed37ac80d30d80f12a18d1b9608b4793d1d25304937b80fdb031cae748a8da2
+  metadata.gz: b1ded185284a923b15342a8f5600e74debdd40f99faf40c35c1b8675636406ff
+  data.tar.gz: '018d94176b65ef8ddcb7b27b785dfaac029ada709e999e58bcc194bff018f984'
 SHA512:
-  metadata.gz: a42d5b9110675cb943a6add8acb5981d81ebbed4827adea17d7d5a819b42dd3e1ae9d0a5cf26288ab4f90f6a6ca28d3325ac431043e50510703aa5fafbef9cac
-  data.tar.gz: 1774c58e9133f64859d84745a39b099b43a0006b331b1926cab3e42ccc8e352c03e1dce990d050253f9c537fb07ed230af10c85558379c68864320a016e1a0e3
+  metadata.gz: f0c47bf01f20c9327b089e60d2bb36d1eee98483eb862e7434b9409a5b18dc24ab38808660b997fbbb66ed21fa012417aaeb1164019ea715c7547d5928ca1d37
+  data.tar.gz: 01e37346bed78463df8a19d1b60315494065094b13aa4f76bbffb872be51f163297542d8a6db2a8d7902c2987312e06f23d3b55e2ed6833f988733c90f11e97c

data/ext/google/protobuf_c/defs.c

@@ -868,6 +868,20 @@ static VALUE FieldDescriptor_default(VALUE _self) {
   return Convert_UpbToRuby(default_val, TypeInfo_get(self->fielddef), Qnil);
 }
 
+
+/*
+ * call-seq:
+ *     FieldDescriptor.json_name => json_name
+ *
+ * Returns this field's json_name, as a Ruby string, or nil if not yet set.
+ */
+static VALUE FieldDescriptor_json_name(VALUE _self) {
+  FieldDescriptor* self = ruby_to_FieldDescriptor(_self);
+  const upb_fielddef *f = self->fielddef;
+  const char *json_name = upb_fielddef_jsonname(f);
+  return rb_str_new2(json_name);
+}
+
 /*
  * call-seq:
  *     FieldDescriptor.label => label
@@ -1043,6 +1057,7 @@ static void FieldDescriptor_register(VALUE module) {
   rb_define_method(klass, "name", FieldDescriptor_name, 0);
   rb_define_method(klass, "type", FieldDescriptor__type, 0);
   rb_define_method(klass, "default", FieldDescriptor_default, 0);
+  rb_define_method(klass, "json_name", FieldDescriptor_json_name, 0);
   rb_define_method(klass, "label", FieldDescriptor_label, 0);
   rb_define_method(klass, "number", FieldDescriptor_number, 0);
   rb_define_method(klass, "submsg_name", FieldDescriptor_submsg_name, 0);
@@ -1750,6 +1765,16 @@ static void msgdef_add_field(VALUE msgbuilder_rb, upb_label_t label, VALUE name,
           field_proto,
           FileBuilderContext_strdup(self->file_builder, default_value));
     }
+
+    if (rb_funcall(options, rb_intern("key?"), 1,
+                   ID2SYM(rb_intern("json_name"))) == Qtrue) {
+      VALUE json_name =
+          rb_hash_lookup(options, ID2SYM(rb_intern("json_name")));
+
+      google_protobuf_FieldDescriptorProto_set_json_name(
+          field_proto,
+          FileBuilderContext_strdup(self->file_builder, json_name));
+    }
   }
 
   if (oneof_index >= 0) {
@@ -1899,18 +1924,20 @@ static VALUE MessageBuilderContext_required(int argc, VALUE* argv,
  */
 static VALUE MessageBuilderContext_repeated(int argc, VALUE* argv,
                                             VALUE _self) {
-  VALUE name, type, number, type_class;
+  VALUE name, type, number;
+  VALUE type_class, options = Qnil;
 
-  if (argc < 3) {
-    rb_raise(rb_eArgError, "Expected at least 3 arguments.");
+  rb_scan_args(argc, argv, "32", &name, &type, &number, &type_class, &options);
+
+  // Allow passing (name, type, number, options) or
+  // (name, type, number, type_class, options)
+  if (argc == 4 && RB_TYPE_P(type_class, T_HASH)) {
+    options = type_class;
+    type_class = Qnil;
   }
-  name = argv[0];
-  type = argv[1];
-  number = argv[2];
-  type_class = (argc > 3) ? argv[3] : Qnil;
 
   msgdef_add_field(_self, UPB_LABEL_REPEATED, name, type, number, type_class,
-                   Qnil, -1, false);
+                   options, -1, false);
 
   return Qnil;
 }

data/ext/google/protobuf_c/extconf.rb

@@ -17,4 +17,5 @@ end
 $objs = ["protobuf.o", "convert.o", "defs.o", "message.o",
          "repeated_field.o", "map.o", "ruby-upb.o", "wrap_memcpy.o"]
 
+find_header('third_party/wyhash/wyhash.h', '../../../..')
 create_makefile("google/protobuf_c")

data/ext/google/protobuf_c/map.c

@@ -167,7 +167,8 @@ VALUE Map_deep_copy(VALUE obj) {
                             new_arena_rb);
 }
 
-const upb_map* Map_GetUpbMap(VALUE val, const upb_fielddef *field) {
+const upb_map* Map_GetUpbMap(VALUE val, const upb_fielddef* field,
+                             upb_arena* arena) {
   const upb_fielddef* key_field = map_field_key(field);
   const upb_fielddef* value_field = map_field_value(field);
   TypeInfo value_type_info = TypeInfo_get(value_field);
@@ -189,6 +190,7 @@ const upb_map* Map_GetUpbMap(VALUE val, const upb_fielddef *field) {
     rb_raise(cTypeError, "Map value type has wrong message/enum class");
   }
 
+  Arena_fuse(self->arena, arena);
   return self->map;
 }
 
@@ -236,7 +238,7 @@ static VALUE Map_merge_into_self(VALUE _self, VALUE hashmap) {
     upb_msg *self_msg = Map_GetMutable(_self);
     size_t iter = UPB_MAP_BEGIN;
 
-    upb_arena_fuse(arena, Arena_get(other->arena));
+    Arena_fuse(other->arena, arena);
 
     if (self->key_type != other->key_type ||
         self->value_type_info.type != other->value_type_info.type ||
@@ -511,7 +513,7 @@ static VALUE Map_dup(VALUE _self) {
   upb_arena *arena = Arena_get(new_self->arena);
   upb_map *new_map = Map_GetMutable(new_map_rb);
 
-  upb_arena_fuse(arena, Arena_get(self->arena));
+  Arena_fuse(self->arena, arena);
 
   while (upb_mapiter_next(self->map, &iter)) {
     upb_msgval key = upb_mapiter_key(self->map, iter);

data/ext/google/protobuf_c/map.h

@@ -44,7 +44,8 @@ VALUE Map_GetRubyWrapper(upb_map *map, upb_fieldtype_t key_type,
 // Gets the underlying upb_map for this Ruby map object, which must have
 // key/value type that match |field|. If this is not a map or the type doesn't
 // match, raises an exception.
-const upb_map *Map_GetUpbMap(VALUE val, const upb_fielddef *field);
+const upb_map *Map_GetUpbMap(VALUE val, const upb_fielddef *field,
+                             upb_arena *arena);
 
 // Implements #inspect for this map by appending its contents to |b|.
 void Map_Inspect(StringBuilder *b, const upb_map *map, upb_fieldtype_t key_type,

data/ext/google/protobuf_c/message.c

@@ -277,9 +277,9 @@ static void Message_setfield(upb_msg* msg, const upb_fielddef* f, VALUE val,
                              upb_arena* arena) {
   upb_msgval msgval;
   if (upb_fielddef_ismap(f)) {
-    msgval.map_val = Map_GetUpbMap(val, f);
+    msgval.map_val = Map_GetUpbMap(val, f, arena);
   } else if (upb_fielddef_isseq(f)) {
-    msgval.array_val = RepeatedField_GetUpbArray(val, f);
+    msgval.array_val = RepeatedField_GetUpbArray(val, f, arena);
   } else {
     if (val == Qnil &&
         (upb_fielddef_issubmsg(f) || upb_fielddef_realcontainingoneof(f))) {
@@ -660,7 +660,7 @@ static VALUE Message_dup(VALUE _self) {
   // TODO(copy unknown fields?)
   // TODO(use official upb msg copy function)
   memcpy((upb_msg*)new_msg_self->msg, self->msg, size);
-  upb_arena_fuse(Arena_get(new_msg_self->arena), Arena_get(self->arena));
+  Arena_fuse(self->arena, Arena_get(new_msg_self->arena));
   return new_msg;
 }
 
@@ -697,16 +697,13 @@ bool Message_Equal(const upb_msg *m1, const upb_msg *m2, const upb_msgdef *m) {
  * field is of a primitive type).
  */
 static VALUE Message_eq(VALUE _self, VALUE _other) {
-  if (TYPE(_self) != TYPE(_other)) {
-    return Qfalse;
-  }
+  if (CLASS_OF(_self) != CLASS_OF(_other)) return Qfalse;
 
   Message* self = ruby_to_Message(_self);
   Message* other = ruby_to_Message(_other);
+  assert(self->msgdef == other->msgdef);
 
-  return Message_Equal(self->msg, other->msg, self->msgdef)
-             ? Qtrue
-             : Qfalse;
+  return Message_Equal(self->msg, other->msg, self->msgdef) ? Qtrue : Qfalse;
 }
 
 uint64_t Message_Hash(const upb_msg* msg, const upb_msgdef* m, uint64_t seed) {
@@ -737,7 +734,10 @@ uint64_t Message_Hash(const upb_msg* msg, const upb_msgdef* m, uint64_t seed) {
  */
 static VALUE Message_hash(VALUE _self) {
   Message* self = ruby_to_Message(_self);
-  return INT2FIX(Message_Hash(self->msg, self->msgdef, 0));
+  uint64_t hash_value = Message_Hash(self->msg, self->msgdef, 0);
+  // RUBY_FIXNUM_MAX should be one less than a power of 2.
+  assert((RUBY_FIXNUM_MAX & (RUBY_FIXNUM_MAX + 1)) == 0);
+  return INT2FIX(hash_value & RUBY_FIXNUM_MAX);
 }
 
 /*
@@ -794,6 +794,14 @@ static VALUE Message_CreateHash(const upb_msg *msg, const upb_msgdef *m) {
     VALUE msg_value;
     VALUE msg_key;
 
+    if (!is_proto2 && upb_fielddef_issubmsg(field) &&
+        !upb_fielddef_isseq(field) && !upb_msg_has(msg, field)) {
+      // TODO: Legacy behavior, remove when we fix the is_proto2 differences.
+      msg_key = ID2SYM(rb_intern(upb_fielddef_name(field)));
+      rb_hash_aset(hash, msg_key, Qnil);
+      continue;
+    }
+
     // Do not include fields that are not present (oneof or optional fields).
     if (is_proto2 && upb_fielddef_haspresence(field) &&
         !upb_msg_has(msg, field)) {
@@ -1306,7 +1314,7 @@ const upb_msg* Message_GetUpbMessage(VALUE value, const upb_msgdef* m,
   }
 
   Message* self = ruby_to_Message(value);
-  upb_arena_fuse(arena, Arena_get(self->arena));
+  Arena_fuse(self->arena, arena);
 
   return self->msg;
 }

data/ext/google/protobuf_c/protobuf.c

@@ -37,7 +37,7 @@
 #include "message.h"
 #include "repeated_field.h"
 
-VALUE cError;
+VALUE cParseError;
 VALUE cTypeError;
 
 const upb_fielddef* map_field_key(const upb_fielddef* field) {
@@ -180,6 +180,7 @@ static void Arena_mark(void *data) {
 static void Arena_free(void *data) {
   Arena *arena = data;
   upb_arena_free(arena->arena);
+  xfree(arena);
 }
 
 static VALUE cArena;
@@ -203,6 +204,16 @@ upb_arena *Arena_get(VALUE _arena) {
   return arena->arena;
 }
 
+void Arena_fuse(VALUE _arena, upb_arena *other) {
+  Arena *arena;
+  TypedData_Get_Struct(_arena, Arena, &Arena_type, arena);
+  if (!upb_arena_fuse(arena->arena, other)) {
+    rb_raise(rb_eRuntimeError,
+             "Unable to fuse arenas. This should never happen since Ruby does "
+             "not use initial blocks");
+  }
+}
+
 VALUE Arena_new() {
   return Arena_alloc(cArena);
 }
@@ -237,8 +248,16 @@ void Arena_register(VALUE module) {
 // We use WeakMap for the cache. For Ruby <2.7 we also need a secondary Hash
 // to store WeakMap keys because Ruby <2.7 WeakMap doesn't allow non-finalizable
 // keys.
-
-#if RUBY_API_VERSION_CODE >= 20700
+//
+// We also need the secondary Hash if sizeof(long) < sizeof(VALUE), because this
+// means it may not be possible to fit a pointer into a Fixnum. Keys are
+// pointers, and if they fit into a Fixnum, Ruby doesn't collect them, but if
+// they overflow and require allocating a Bignum, they could get collected
+// prematurely, thus removing the cache entry. This happens on 64-bit Windows,
+// on which pointers are 64 bits but longs are 32 bits. In this case, we enable
+// the secondary Hash to hold the keys and prevent them from being collected.
+
+#if RUBY_API_VERSION_CODE >= 20700 && SIZEOF_LONG >= SIZEOF_VALUE
 #define USE_SECONDARY_MAP 0
 #else
 #define USE_SECONDARY_MAP 1
@@ -251,15 +270,81 @@ void Arena_register(VALUE module) {
 // The object is used only for its identity; it does not contain any data.
 VALUE secondary_map = Qnil;
 
+// Mutations to the map are under a mutex, because SeconaryMap_MaybeGC()
+// iterates over the map which cannot happen in parallel with insertions, or
+// Ruby will throw:
+//   can't add a new key into hash during iteration (RuntimeError)
+VALUE secondary_map_mutex = Qnil;
+
+// Lambda that will GC entries from the secondary map that are no longer present
+// in the primary map.
+VALUE gc_secondary_map_lambda = Qnil;
+ID length;
+
+extern VALUE weak_obj_cache;
+
 static void SecondaryMap_Init() {
   rb_gc_register_address(&secondary_map);
+  rb_gc_register_address(&gc_secondary_map_lambda);
+  rb_gc_register_address(&secondary_map_mutex);
   secondary_map = rb_hash_new();
+  gc_secondary_map_lambda = rb_eval_string(
+      "->(secondary, weak) {\n"
+      "  secondary.delete_if { |k, v| !weak.key?(v) }\n"
+      "}\n");
+  secondary_map_mutex = rb_mutex_new();
+  length = rb_intern("length");
 }
 
-static VALUE SecondaryMap_Get(VALUE key) {
+// The secondary map is a regular Hash, and will never shrink on its own.
+// The main object cache is a WeakMap that will automatically remove entries
+// when the target object is no longer reachable, but unless we manually
+// remove the corresponding entries from the secondary map, it will grow
+// without bound.
+//
+// To avoid this unbounded growth we periodically remove entries from the
+// secondary map that are no longer present in the WeakMap. The logic of
+// how often to perform this GC is an artbirary tuning parameter that
+// represents a straightforward CPU/memory tradeoff.
+//
+// Requires: secondary_map_mutex is held.
+static void SecondaryMap_MaybeGC() {
+  PBRUBY_ASSERT(rb_mutex_locked_p(secondary_map_mutex) == Qtrue);
+  size_t weak_len = NUM2ULL(rb_funcall(weak_obj_cache, length, 0));
+  size_t secondary_len = RHASH_SIZE(secondary_map);
+  if (secondary_len < weak_len) {
+    // Logically this case should not be possible: a valid entry cannot exist in
+    // the weak table unless there is a corresponding entry in the secondary
+    // table. It should *always* be the case that secondary_len >= weak_len.
+    //
+    // However ObjectSpace::WeakMap#length (and therefore weak_len) is
+    // unreliable: it overreports its true length by including non-live objects.
+    // However these non-live objects are not yielded in iteration, so we may
+    // have previously deleted them from the secondary map in a previous
+    // invocation of SecondaryMap_MaybeGC().
+    //
+    // In this case, we can't measure any waste, so we just return.
+    return;
+  }
+  size_t waste = secondary_len - weak_len;
+  // GC if we could remove at least 2000 entries or 20% of the table size
+  // (whichever is greater).  Since the cost of the GC pass is O(N), we
+  // want to make sure that we condition this on overall table size, to
+  // avoid O(N^2) CPU costs.
+  size_t threshold = PBRUBY_MAX(secondary_len * 0.2, 2000);
+  if (waste > threshold) {
+    rb_funcall(gc_secondary_map_lambda, rb_intern("call"), 2,
+               secondary_map, weak_obj_cache);
+  }
+}
+
+// Requires: secondary_map_mutex is held by this thread iff create == true.
+static VALUE SecondaryMap_Get(VALUE key, bool create) {
+  PBRUBY_ASSERT(!create || rb_mutex_locked_p(secondary_map_mutex) == Qtrue);
   VALUE ret = rb_hash_lookup(secondary_map, key);
-  if (ret == Qnil) {
-    ret = rb_eval_string("Object.new");
+  if (ret == Qnil && create) {
+    SecondaryMap_MaybeGC();
+    ret = rb_class_new_instance(0, NULL, rb_cObject);
     rb_hash_aset(secondary_map, key, ret);
   }
   return ret;
@@ -267,14 +352,13 @@ static VALUE SecondaryMap_Get(VALUE key) {
 
 #endif
 
-static VALUE ObjectCache_GetKey(const void* key) {
-  char buf[sizeof(key)];
-  memcpy(&buf, &key, sizeof(key));
-  intptr_t key_int = (intptr_t)key;
-  PBRUBY_ASSERT((key_int & 3) == 0);
-  VALUE ret = LL2NUM(key_int >> 2);
+// Requires: secondary_map_mutex is held by this thread iff create == true.
+static VALUE ObjectCache_GetKey(const void* key, bool create) {
+  VALUE key_val = (VALUE)key;
+  PBRUBY_ASSERT((key_val & 3) == 0);
+  VALUE ret = LL2NUM(key_val >> 2);
 #if USE_SECONDARY_MAP
-  ret = SecondaryMap_Get(ret);
+  ret = SecondaryMap_Get(ret, create);
 #endif
   return ret;
 }
@@ -298,14 +382,20 @@ static void ObjectCache_Init() {
 
 void ObjectCache_Add(const void* key, VALUE val) {
   PBRUBY_ASSERT(ObjectCache_Get(key) == Qnil);
-  VALUE key_rb = ObjectCache_GetKey(key);
+#if USE_SECONDARY_MAP
+  rb_mutex_lock(secondary_map_mutex);
+#endif
+  VALUE key_rb = ObjectCache_GetKey(key, true);
   rb_funcall(weak_obj_cache, item_set, 2, key_rb, val);
+#if USE_SECONDARY_MAP
+  rb_mutex_unlock(secondary_map_mutex);
+#endif
   PBRUBY_ASSERT(ObjectCache_Get(key) == val);
 }
 
 // Returns the cached object for this key, if any. Otherwise returns Qnil.
 VALUE ObjectCache_Get(const void* key) {
-  VALUE key_rb = ObjectCache_GetKey(key);
+  VALUE key_rb = ObjectCache_GetKey(key, false);
   return rb_funcall(weak_obj_cache, item_get, 1, key_rb);
 }
 
@@ -368,8 +458,10 @@ void Init_protobuf_c() {
   Map_register(protobuf);
   Message_register(protobuf);
 
-  cError = rb_const_get(protobuf, rb_intern("Error"));
+  cParseError = rb_const_get(protobuf, rb_intern("ParseError"));
+  rb_gc_register_mark_object(cParseError);
   cTypeError = rb_const_get(protobuf, rb_intern("TypeError"));
+  rb_gc_register_mark_object(cTypeError);
 
   rb_define_singleton_method(protobuf, "discard_unknown",
                              Google_Protobuf_discard_unknown, 1);

data/ext/google/protobuf_c/protobuf.h

@@ -55,6 +55,10 @@ const upb_fielddef* map_field_value(const upb_fielddef* field);
 VALUE Arena_new();
 upb_arena *Arena_get(VALUE arena);
 
+// Fuses this arena to another, throwing a Ruby exception if this is not
+// possible.
+void Arena_fuse(VALUE arena, upb_arena *other);
+
 // Pins this Ruby object to the lifetime of this arena, so that as long as the
 // arena is alive this object will not be collected.
 //
@@ -106,6 +110,8 @@ extern VALUE cTypeError;
 #define PBRUBY_ASSERT(expr) assert(expr)
 #endif
 
+#define PBRUBY_MAX(x, y) (((x) > (y)) ? (x) : (y))
+
 #define UPB_UNUSED(var) (void)var
 
 #endif  // __GOOGLE_PROTOBUF_RUBY_PROTOBUF_H__

data/ext/google/protobuf_c/repeated_field.c

@@ -149,7 +149,8 @@ VALUE RepeatedField_deep_copy(VALUE _self) {
   return new_rptfield;
 }
 
-const upb_array* RepeatedField_GetUpbArray(VALUE val, const upb_fielddef *field) {
+const upb_array* RepeatedField_GetUpbArray(VALUE val, const upb_fielddef* field,
+                                           upb_arena* arena) {
   RepeatedField* self;
   TypeInfo type_info = TypeInfo_get(field);
 
@@ -167,6 +168,7 @@ const upb_array* RepeatedField_GetUpbArray(VALUE val, const upb_fielddef *field)
     rb_raise(cTypeError, "Repeated field array has wrong message/enum class");
   }
 
+  Arena_fuse(self->arena, arena);
   return self->array;
 }
 
@@ -412,7 +414,7 @@ static VALUE RepeatedField_dup(VALUE _self) {
   int size = upb_array_size(self->array);
   int i;
 
-  upb_arena_fuse(arena, Arena_get(self->arena));
+  Arena_fuse(self->arena, arena);
 
   for (i = 0; i < size; i++) {
     upb_msgval msgval = upb_array_get(self->array, i);

data/ext/google/protobuf_c/repeated_field.h

@@ -44,7 +44,8 @@ VALUE RepeatedField_GetRubyWrapper(upb_array* msg, TypeInfo type_info,
 // Gets the underlying upb_array for this Ruby RepeatedField object, which must
 // have a type that matches |f|. If this is not a repeated field or the type
 // doesn't match, raises an exception.
-const upb_array* RepeatedField_GetUpbArray(VALUE value, const upb_fielddef* f);
+const upb_array* RepeatedField_GetUpbArray(VALUE value, const upb_fielddef* f,
+                                           upb_arena* arena);
 
 // Implements #inspect for this repeated field by appending its contents to |b|.
 void RepeatedField_Inspect(StringBuilder* b, const upb_array* array,