google-protobuf 3.15.7 → 3.19.0

data/ext/google/protobuf_c/map.c

@@ -167,7 +167,8 @@ VALUE Map_deep_copy(VALUE obj) {
                             new_arena_rb);
 }
 
-const upb_map* Map_GetUpbMap(VALUE val, const upb_fielddef *field) {
+const upb_map* Map_GetUpbMap(VALUE val, const upb_fielddef* field,
+                             upb_arena* arena) {
   const upb_fielddef* key_field = map_field_key(field);
   const upb_fielddef* value_field = map_field_value(field);
   TypeInfo value_type_info = TypeInfo_get(value_field);
@@ -189,6 +190,7 @@ const upb_map* Map_GetUpbMap(VALUE val, const upb_fielddef *field) {
     rb_raise(cTypeError, "Map value type has wrong message/enum class");
   }
 
+  Arena_fuse(self->arena, arena);
   return self->map;
 }
 
@@ -236,7 +238,7 @@ static VALUE Map_merge_into_self(VALUE _self, VALUE hashmap) {
     upb_msg *self_msg = Map_GetMutable(_self);
     size_t iter = UPB_MAP_BEGIN;
 
-    upb_arena_fuse(arena, Arena_get(other->arena));
+    Arena_fuse(other->arena, arena);
 
     if (self->key_type != other->key_type ||
         self->value_type_info.type != other->value_type_info.type ||
@@ -511,7 +513,7 @@ static VALUE Map_dup(VALUE _self) {
   upb_arena *arena = Arena_get(new_self->arena);
   upb_map *new_map = Map_GetMutable(new_map_rb);
 
-  upb_arena_fuse(arena, Arena_get(self->arena));
+  Arena_fuse(self->arena, arena);
 
   while (upb_mapiter_next(self->map, &iter)) {
     upb_msgval key = upb_mapiter_key(self->map, iter);
@@ -678,7 +680,10 @@ void Map_register(VALUE module) {
   rb_define_method(klass, "delete", Map_delete, 1);
   rb_define_method(klass, "clear", Map_clear, 0);
   rb_define_method(klass, "length", Map_length, 0);
+  rb_define_method(klass, "size", Map_length, 0);
   rb_define_method(klass, "dup", Map_dup, 0);
+  // Also define #clone so that we don't inherit Object#clone.
+  rb_define_method(klass, "clone", Map_dup, 0);
   rb_define_method(klass, "==", Map_eq, 1);
   rb_define_method(klass, "freeze", Map_freeze, 0);
   rb_define_method(klass, "hash", Map_hash, 0);

data/ext/google/protobuf_c/map.h

@@ -44,7 +44,8 @@ VALUE Map_GetRubyWrapper(upb_map *map, upb_fieldtype_t key_type,
 // Gets the underlying upb_map for this Ruby map object, which must have
 // key/value type that match |field|. If this is not a map or the type doesn't
 // match, raises an exception.
-const upb_map *Map_GetUpbMap(VALUE val, const upb_fielddef *field);
+const upb_map *Map_GetUpbMap(VALUE val, const upb_fielddef *field,
+                             upb_arena *arena);
 
 // Implements #inspect for this map by appending its contents to |b|.
 void Map_Inspect(StringBuilder *b, const upb_map *map, upb_fieldtype_t key_type,

data/ext/google/protobuf_c/message.c

@@ -35,7 +35,6 @@
 #include "map.h"
 #include "protobuf.h"
 #include "repeated_field.h"
-#include "third_party/wyhash/wyhash.h"
 
 static VALUE cParseError = Qnil;
 static ID descriptor_instancevar_interned;
@@ -277,9 +276,9 @@ static void Message_setfield(upb_msg* msg, const upb_fielddef* f, VALUE val,
                              upb_arena* arena) {
   upb_msgval msgval;
   if (upb_fielddef_ismap(f)) {
-    msgval.map_val = Map_GetUpbMap(val, f);
+    msgval.map_val = Map_GetUpbMap(val, f, arena);
   } else if (upb_fielddef_isseq(f)) {
-    msgval.array_val = RepeatedField_GetUpbArray(val, f);
+    msgval.array_val = RepeatedField_GetUpbArray(val, f, arena);
   } else {
     if (val == Qnil &&
         (upb_fielddef_issubmsg(f) || upb_fielddef_realcontainingoneof(f))) {
@@ -660,7 +659,7 @@ static VALUE Message_dup(VALUE _self) {
   // TODO(copy unknown fields?)
   // TODO(use official upb msg copy function)
   memcpy((upb_msg*)new_msg_self->msg, self->msg, size);
-  upb_arena_fuse(Arena_get(new_msg_self->arena), Arena_get(self->arena));
+  Arena_fuse(self->arena, Arena_get(new_msg_self->arena));
   return new_msg;
 }
 
@@ -717,7 +716,7 @@ uint64_t Message_Hash(const upb_msg* msg, const upb_msgdef* m, uint64_t seed) {
                        &size);
 
   if (data) {
-    uint64_t ret = wyhash(data, size, seed, _wyp);
+    uint64_t ret = Wyhash(data, size, seed, kWyhashSalt);
     upb_arena_free(arena);
     return ret;
   } else {
@@ -734,7 +733,10 @@ uint64_t Message_Hash(const upb_msg* msg, const upb_msgdef* m, uint64_t seed) {
  */
 static VALUE Message_hash(VALUE _self) {
   Message* self = ruby_to_Message(_self);
-  return INT2FIX(Message_Hash(self->msg, self->msgdef, 0));
+  uint64_t hash_value = Message_Hash(self->msg, self->msgdef, 0);
+  // RUBY_FIXNUM_MAX should be one less than a power of 2.
+  assert((RUBY_FIXNUM_MAX & (RUBY_FIXNUM_MAX + 1)) == 0);
+  return INT2FIX(hash_value & RUBY_FIXNUM_MAX);
 }
 
 /*
@@ -791,6 +793,14 @@ static VALUE Message_CreateHash(const upb_msg *msg, const upb_msgdef *m) {
     VALUE msg_value;
     VALUE msg_key;
 
+    if (!is_proto2 && upb_fielddef_issubmsg(field) &&
+        !upb_fielddef_isseq(field) && !upb_msg_has(msg, field)) {
+      // TODO: Legacy behavior, remove when we fix the is_proto2 differences.
+      msg_key = ID2SYM(rb_intern(upb_fielddef_name(field)));
+      rb_hash_aset(hash, msg_key, Qnil);
+      continue;
+    }
+
     // Do not include fields that are not present (oneof or optional fields).
     if (is_proto2 && upb_fielddef_haspresence(field) &&
         !upb_msg_has(msg, field)) {
@@ -1303,7 +1313,7 @@ const upb_msg* Message_GetUpbMessage(VALUE value, const upb_msgdef* m,
   }
 
   Message* self = ruby_to_Message(value);
-  upb_arena_fuse(arena, Arena_get(self->arena));
+  Arena_fuse(self->arena, arena);
 
   return self->msg;
 }

data/ext/google/protobuf_c/protobuf.c

@@ -37,7 +37,7 @@
 #include "message.h"
 #include "repeated_field.h"
 
-VALUE cError;
+VALUE cParseError;
 VALUE cTypeError;
 
 const upb_fielddef* map_field_key(const upb_fielddef* field) {
@@ -180,6 +180,7 @@ static void Arena_mark(void *data) {
 static void Arena_free(void *data) {
   Arena *arena = data;
   upb_arena_free(arena->arena);
+  xfree(arena);
 }
 
 static VALUE cArena;
@@ -203,6 +204,16 @@ upb_arena *Arena_get(VALUE _arena) {
   return arena->arena;
 }
 
+void Arena_fuse(VALUE _arena, upb_arena *other) {
+  Arena *arena;
+  TypedData_Get_Struct(_arena, Arena, &Arena_type, arena);
+  if (!upb_arena_fuse(arena->arena, other)) {
+    rb_raise(rb_eRuntimeError,
+             "Unable to fuse arenas. This should never happen since Ruby does "
+             "not use initial blocks");
+  }
+}
+
 VALUE Arena_new() {
   return Arena_alloc(cArena);
 }
@@ -237,8 +248,16 @@ void Arena_register(VALUE module) {
 // We use WeakMap for the cache. For Ruby <2.7 we also need a secondary Hash
 // to store WeakMap keys because Ruby <2.7 WeakMap doesn't allow non-finalizable
 // keys.
-
-#if RUBY_API_VERSION_CODE >= 20700
+//
+// We also need the secondary Hash if sizeof(long) < sizeof(VALUE), because this
+// means it may not be possible to fit a pointer into a Fixnum. Keys are
+// pointers, and if they fit into a Fixnum, Ruby doesn't collect them, but if
+// they overflow and require allocating a Bignum, they could get collected
+// prematurely, thus removing the cache entry. This happens on 64-bit Windows,
+// on which pointers are 64 bits but longs are 32 bits. In this case, we enable
+// the secondary Hash to hold the keys and prevent them from being collected.
+
+#if RUBY_API_VERSION_CODE >= 20700 && SIZEOF_LONG >= SIZEOF_VALUE
 #define USE_SECONDARY_MAP 0
 #else
 #define USE_SECONDARY_MAP 1
@@ -325,7 +344,7 @@ static VALUE SecondaryMap_Get(VALUE key, bool create) {
   VALUE ret = rb_hash_lookup(secondary_map, key);
   if (ret == Qnil && create) {
     SecondaryMap_MaybeGC();
-    ret = rb_eval_string("Object.new");
+    ret = rb_class_new_instance(0, NULL, rb_cObject);
     rb_hash_aset(secondary_map, key, ret);
   }
   return ret;
@@ -335,11 +354,9 @@ static VALUE SecondaryMap_Get(VALUE key, bool create) {
 
 // Requires: secondary_map_mutex is held by this thread iff create == true.
 static VALUE ObjectCache_GetKey(const void* key, bool create) {
-  char buf[sizeof(key)];
-  memcpy(&buf, &key, sizeof(key));
-  intptr_t key_int = (intptr_t)key;
-  PBRUBY_ASSERT((key_int & 3) == 0);
-  VALUE ret = LL2NUM(key_int >> 2);
+  VALUE key_val = (VALUE)key;
+  PBRUBY_ASSERT((key_val & 3) == 0);
+  VALUE ret = LL2NUM(key_val >> 2);
 #if USE_SECONDARY_MAP
   ret = SecondaryMap_Get(ret, create);
 #endif
@@ -441,8 +458,10 @@ void Init_protobuf_c() {
   Map_register(protobuf);
   Message_register(protobuf);
 
-  cError = rb_const_get(protobuf, rb_intern("Error"));
+  cParseError = rb_const_get(protobuf, rb_intern("ParseError"));
+  rb_gc_register_mark_object(cParseError);
   cTypeError = rb_const_get(protobuf, rb_intern("TypeError"));
+  rb_gc_register_mark_object(cTypeError);
 
   rb_define_singleton_method(protobuf, "discard_unknown",
                              Google_Protobuf_discard_unknown, 1);

data/ext/google/protobuf_c/protobuf.h

@@ -55,6 +55,10 @@ const upb_fielddef* map_field_value(const upb_fielddef* field);
 VALUE Arena_new();
 upb_arena *Arena_get(VALUE arena);
 
+// Fuses this arena to another, throwing a Ruby exception if this is not
+// possible.
+void Arena_fuse(VALUE arena, upb_arena *other);
+
 // Pins this Ruby object to the lifetime of this arena, so that as long as the
 // arena is alive this object will not be collected.
 //

data/ext/google/protobuf_c/repeated_field.c

@@ -34,7 +34,6 @@
 #include "defs.h"
 #include "message.h"
 #include "protobuf.h"
-#include "third_party/wyhash/wyhash.h"
 
 // -----------------------------------------------------------------------------
 // Repeated field container type.
@@ -149,7 +148,8 @@ VALUE RepeatedField_deep_copy(VALUE _self) {
   return new_rptfield;
 }
 
-const upb_array* RepeatedField_GetUpbArray(VALUE val, const upb_fielddef *field) {
+const upb_array* RepeatedField_GetUpbArray(VALUE val, const upb_fielddef* field,
+                                           upb_arena* arena) {
   RepeatedField* self;
   TypeInfo type_info = TypeInfo_get(field);
 
@@ -167,6 +167,7 @@ const upb_array* RepeatedField_GetUpbArray(VALUE val, const upb_fielddef *field)
     rb_raise(cTypeError, "Repeated field array has wrong message/enum class");
   }
 
+  Arena_fuse(self->arena, arena);
   return self->array;
 }
 
@@ -412,7 +413,7 @@ static VALUE RepeatedField_dup(VALUE _self) {
   int size = upb_array_size(self->array);
   int i;
 
-  upb_arena_fuse(arena, Arena_get(self->arena));
+  Arena_fuse(self->arena, arena);
 
   for (i = 0; i < size; i++) {
     upb_msgval msgval = upb_array_get(self->array, i);
@@ -550,6 +551,7 @@ VALUE RepeatedField_plus(VALUE _self, VALUE list) {
     RepeatedField* dupped = ruby_to_RepeatedField(dupped_);
     upb_array *dupped_array = RepeatedField_GetMutable(dupped_);
     upb_arena* arena = Arena_get(dupped->arena);
+    Arena_fuse(list_rptfield->arena, arena);
     int size = upb_array_size(list_rptfield->array);
     int i;
 

data/ext/google/protobuf_c/repeated_field.h

@@ -44,7 +44,8 @@ VALUE RepeatedField_GetRubyWrapper(upb_array* msg, TypeInfo type_info,
 // Gets the underlying upb_array for this Ruby RepeatedField object, which must
 // have a type that matches |f|. If this is not a repeated field or the type
 // doesn't match, raises an exception.
-const upb_array* RepeatedField_GetUpbArray(VALUE value, const upb_fielddef* f);
+const upb_array* RepeatedField_GetUpbArray(VALUE value, const upb_fielddef* f,
+                                           upb_arena* arena);
 
 // Implements #inspect for this repeated field by appending its contents to |b|.
 void RepeatedField_Inspect(StringBuilder* b, const upb_array* array,