google-protobuf 3.8.0 → 3.15.0

data/ext/google/protobuf_c/message.h

@@ -0,0 +1,98 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef RUBY_PROTOBUF_MESSAGE_H_
+#define RUBY_PROTOBUF_MESSAGE_H_
+
+#include <ruby/ruby.h>
+
+#include "protobuf.h"
+#include "ruby-upb.h"
+
+// Gets the underlying upb_msg* and upb_msgdef for the given Ruby message
+// wrapper. Requires that |value| is indeed a message object.
+const upb_msg *Message_Get(VALUE value, const upb_msgdef **m);
+
+// Like Message_Get(), but checks that the object is not frozen and returns a
+// mutable pointer.
+upb_msg *Message_GetMutable(VALUE value, const upb_msgdef **m);
+
+// Returns the Arena object for this message.
+VALUE Message_GetArena(VALUE value);
+
+// Converts |value| into a upb_msg value of the expected upb_msgdef type,
+// raising an error if this is not possible. Used when assigning |value| to a
+// field of another message, which means the message must be of a particular
+// type.
+//
+// This will perform automatic conversions in some cases (for example, Time ->
+// Google::Protobuf::Timestamp). If any new message is created, it will be
+// created on |arena|, and any existing message will have its arena fused with
+// |arena|.
+const upb_msg* Message_GetUpbMessage(VALUE value, const upb_msgdef* m,
+                                     const char* name, upb_arena* arena);
+
+// Gets or constructs a Ruby wrapper object for the given message. The wrapper
+// object will reference |arena| and ensure that it outlives this object.
+VALUE Message_GetRubyWrapper(upb_msg* msg, const upb_msgdef* m, VALUE arena);
+
+// Implements #inspect for this message, printing the text to |b|.
+void Message_PrintMessage(StringBuilder* b, const upb_msg* msg,
+                          const upb_msgdef* m);
+
+// Returns a hash value for the given message.
+uint64_t Message_Hash(const upb_msg *msg, const upb_msgdef *m, uint64_t seed);
+
+// Returns a deep copy of the given message.
+upb_msg* Message_deep_copy(const upb_msg* msg, const upb_msgdef* m,
+                           upb_arena *arena);
+
+// Returns true if these two messages are equal.
+bool Message_Equal(const upb_msg *m1, const upb_msg *m2, const upb_msgdef *m);
+
+// Checks that this Ruby object is a message, and raises an exception if not.
+void Message_CheckClass(VALUE klass);
+
+// Returns a new Hash object containing the contents of this message.
+VALUE Scalar_CreateHash(upb_msgval val, TypeInfo type_info);
+
+// Creates a message class or enum module for this descriptor, respectively.
+VALUE build_class_from_descriptor(VALUE descriptor);
+VALUE build_module_from_enumdesc(VALUE _enumdesc);
+
+// Returns the Descriptor/EnumDescriptor for the given message class or enum
+// module, respectively. Returns nil if this is not a message class or enum
+// module.
+VALUE MessageOrEnum_GetDescriptor(VALUE klass);
+
+// Call at startup to register all types in this module.
+void Message_register(VALUE protobuf);
+
+#endif  // RUBY_PROTOBUF_MESSAGE_H_

data/ext/google/protobuf_c/protobuf.c

@@ -30,52 +30,342 @@
 
 #include "protobuf.h"
 
-// -----------------------------------------------------------------------------
-// Global map from upb {msg,enum}defs to wrapper Descriptor/EnumDescriptor
-// instances.
-// -----------------------------------------------------------------------------
+#include <ruby/version.h>
 
-// This is a hash table from def objects (encoded by converting pointers to
-// Ruby integers) to MessageDef/EnumDef instances (as Ruby values).
-VALUE upb_def_to_ruby_obj_map;
+#include "defs.h"
+#include "map.h"
+#include "message.h"
+#include "repeated_field.h"
 
 VALUE cError;
-VALUE cParseError;
 VALUE cTypeError;
 
-void add_def_obj(const void* def, VALUE value) {
-  rb_hash_aset(upb_def_to_ruby_obj_map, ULL2NUM((intptr_t)def), value);
+const upb_fielddef* map_field_key(const upb_fielddef* field) {
+  const upb_msgdef *entry = upb_fielddef_msgsubdef(field);
+  return upb_msgdef_itof(entry, 1);
 }
 
-VALUE get_def_obj(const void* def) {
-  return rb_hash_aref(upb_def_to_ruby_obj_map, ULL2NUM((intptr_t)def));
+const upb_fielddef* map_field_value(const upb_fielddef* field) {
+  const upb_msgdef *entry = upb_fielddef_msgsubdef(field);
+  return upb_msgdef_itof(entry, 2);
 }
 
 // -----------------------------------------------------------------------------
-// Utilities.
+// StringBuilder, for inspect
 // -----------------------------------------------------------------------------
 
-// Raises a Ruby error if |status| is not OK, using its error message.
-void check_upb_status(const upb_status* status, const char* msg) {
-  if (!upb_ok(status)) {
-    rb_raise(rb_eRuntimeError, "%s: %s\n", msg, upb_status_errmsg(status));
+struct StringBuilder {
+  size_t size;
+  size_t cap;
+  char *data;
+};
+
+typedef struct StringBuilder StringBuilder;
+
+static size_t StringBuilder_SizeOf(size_t cap) {
+  return sizeof(StringBuilder) + cap;
+}
+
+StringBuilder* StringBuilder_New() {
+  const size_t cap = 128;
+  StringBuilder* builder = malloc(sizeof(*builder));
+  builder->size = 0;
+  builder->cap = cap;
+  builder->data = malloc(builder->cap);
+  return builder;
+}
+
+void StringBuilder_Free(StringBuilder* b) {
+  free(b->data);
+  free(b);
+}
+
+void StringBuilder_Printf(StringBuilder* b, const char *fmt, ...) {
+  size_t have = b->cap - b->size;
+  size_t n;
+  va_list args;
+
+  va_start(args, fmt);
+  n = vsnprintf(&b->data[b->size], have, fmt, args);
+  va_end(args);
+
+  if (have <= n) {
+    while (have <= n) {
+      b->cap *= 2;
+      have = b->cap - b->size;
+    }
+    b->data = realloc(b->data, StringBuilder_SizeOf(b->cap));
+    va_start(args, fmt);
+    n = vsnprintf(&b->data[b->size], have, fmt, args);
+    va_end(args);
+    PBRUBY_ASSERT(n < have);
+  }
+
+  b->size += n;
+}
+
+VALUE StringBuilder_ToRubyString(StringBuilder* b) {
+  VALUE ret = rb_str_new(b->data, b->size);
+  rb_enc_associate(ret, rb_utf8_encoding());
+  return ret;
+}
+
+static void StringBuilder_PrintEnum(StringBuilder* b, int32_t val,
+                                    const upb_enumdef* e) {
+  const char *name = upb_enumdef_iton(e, val);
+  if (name) {
+    StringBuilder_Printf(b, ":%s", name);
+  } else {
+    StringBuilder_Printf(b, "%" PRId32, val);
+  }
+}
+
+void StringBuilder_PrintMsgval(StringBuilder* b, upb_msgval val,
+                               TypeInfo info) {
+  switch (info.type) {
+    case UPB_TYPE_BOOL:
+      StringBuilder_Printf(b, "%s", val.bool_val ? "true" : "false");
+      break;
+    case UPB_TYPE_FLOAT: {
+      VALUE str = rb_inspect(DBL2NUM(val.float_val));
+      StringBuilder_Printf(b, "%s", RSTRING_PTR(str));
+      break;
+    }
+    case UPB_TYPE_DOUBLE: {
+      VALUE str = rb_inspect(DBL2NUM(val.double_val));
+      StringBuilder_Printf(b, "%s", RSTRING_PTR(str));
+      break;
+    }
+    case UPB_TYPE_INT32:
+      StringBuilder_Printf(b, "%" PRId32, val.int32_val);
+      break;
+    case UPB_TYPE_UINT32:
+      StringBuilder_Printf(b, "%" PRIu32, val.uint32_val);
+      break;
+    case UPB_TYPE_INT64:
+      StringBuilder_Printf(b, "%" PRId64, val.int64_val);
+      break;
+    case UPB_TYPE_UINT64:
+      StringBuilder_Printf(b, "%" PRIu64, val.uint64_val);
+      break;
+    case UPB_TYPE_STRING:
+      StringBuilder_Printf(b, "\"%.*s\"", (int)val.str_val.size, val.str_val.data);
+      break;
+    case UPB_TYPE_BYTES:
+      StringBuilder_Printf(b, "\"%.*s\"", (int)val.str_val.size, val.str_val.data);
+      break;
+    case UPB_TYPE_ENUM:
+      StringBuilder_PrintEnum(b, val.int32_val, info.def.enumdef);
+      break;
+    case UPB_TYPE_MESSAGE:
+      Message_PrintMessage(b, val.msg_val, info.def.msgdef);
+      break;
   }
 }
 
-// String encodings: we look these up once, at load time, and then cache them
-// here.
-rb_encoding* kRubyStringUtf8Encoding;
-rb_encoding* kRubyStringASCIIEncoding;
-rb_encoding* kRubyString8bitEncoding;
+// -----------------------------------------------------------------------------
+// Arena
+// -----------------------------------------------------------------------------
+
+void Arena_free(void* data) { upb_arena_free(data); }
+
+static VALUE cArena;
+
+const rb_data_type_t Arena_type = {
+  "Google::Protobuf::Internal::Arena",
+  { NULL, Arena_free, NULL },
+};
+
+static VALUE Arena_alloc(VALUE klass) {
+  upb_arena *arena = upb_arena_new();
+  return TypedData_Wrap_Struct(klass, &Arena_type, arena);
+}
+
+upb_arena *Arena_get(VALUE _arena) {
+  upb_arena *arena;
+  TypedData_Get_Struct(_arena, upb_arena, &Arena_type, arena);
+  return arena;
+}
+
+VALUE Arena_new() {
+  return Arena_alloc(cArena);
+}
+
+void Arena_register(VALUE module) {
+  VALUE internal = rb_define_module_under(module, "Internal");
+  VALUE klass = rb_define_class_under(internal, "Arena", rb_cObject);
+  rb_define_alloc_func(klass, Arena_alloc);
+  rb_gc_register_address(&cArena);
+  cArena = klass;
+}
+
+// -----------------------------------------------------------------------------
+// Object Cache
+// -----------------------------------------------------------------------------
 
-// Ruby-interned string: "descriptor". We use this identifier to store an
-// instance variable on message classes we create in order to link them back to
-// their descriptors.
+// A pointer -> Ruby Object cache that keeps references to Ruby wrapper
+// objects.  This allows us to look up any Ruby wrapper object by the address
+// of the object it is wrapping. That way we can avoid ever creating two
+// different wrapper objects for the same C object, which saves memory and
+// preserves object identity.
 //
-// We intern this once at module load time then use the interned identifier at
-// runtime in order to avoid the cost of repeatedly interning in hot paths.
-const char* kDescriptorInstanceVar = "descriptor";
-ID descriptor_instancevar_interned;
+// We use Hash and/or WeakMap for the cache. WeakMap is faster overall
+// (probably due to removal being integrated with GC) but doesn't work for Ruby
+// <2.7 (see note below). We need Hash for Ruby <2.7 and for cases where we
+// need to GC-root the object (notably when the object has been frozen).
+
+#if RUBY_API_VERSION_CODE >= 20700
+#define USE_WEAK_MAP 1
+#else
+#define USE_WEAK_MAP 0
+#endif
+
+static VALUE ObjectCache_GetKey(const void* key) {
+  char buf[sizeof(key)];
+  memcpy(&buf, &key, sizeof(key));
+  intptr_t key_int = (intptr_t)key;
+  PBRUBY_ASSERT((key_int & 3) == 0);
+  return LL2NUM(key_int >> 2);
+}
+
+// Strong object cache, uses regular Hash and GC-roots objects.
+// - For Ruby <2.7, used for all objects.
+// - For Ruby >=2.7, used only for frozen objects, so we preserve the "frozen"
+//   bit (since this information is not preserved at the upb level).
+
+VALUE strong_obj_cache = Qnil;
+
+static void StrongObjectCache_Init() {
+  rb_gc_register_address(&strong_obj_cache);
+  strong_obj_cache = rb_hash_new();
+}
+
+static void StrongObjectCache_Remove(void* key) {
+  VALUE key_rb = ObjectCache_GetKey(key);
+  PBRUBY_ASSERT(rb_hash_lookup(strong_obj_cache, key_rb) != Qnil);
+  rb_hash_delete(strong_obj_cache, key_rb);
+}
+
+static VALUE StrongObjectCache_Get(const void* key) {
+  VALUE key_rb = ObjectCache_GetKey(key);
+  return rb_hash_lookup(strong_obj_cache, key_rb);
+}
+
+static void StrongObjectCache_Add(const void* key, VALUE val,
+                                  upb_arena* arena) {
+  PBRUBY_ASSERT(StrongObjectCache_Get(key) == Qnil);
+  VALUE key_rb = ObjectCache_GetKey(key);
+  rb_hash_aset(strong_obj_cache, key_rb, val);
+  upb_arena_addcleanup(arena, (void*)key, StrongObjectCache_Remove);
+}
+
+// Weak object cache. This speeds up the test suite significantly, so we
+// presume it speeds up real code also. However we can only use it in Ruby
+// >=2.7 due to:
+//   https://bugs.ruby-lang.org/issues/16035
+
+#if USE_WEAK_MAP
+
+VALUE weak_obj_cache = Qnil;
+
+static void WeakObjectCache_Init() {
+  rb_gc_register_address(&weak_obj_cache);
+  VALUE klass = rb_eval_string("ObjectSpace::WeakMap");
+  weak_obj_cache = rb_class_new_instance(0, NULL, klass);
+}
+
+static VALUE WeakObjectCache_Get(const void* key) {
+  VALUE key_rb = ObjectCache_GetKey(key);
+  VALUE ret = rb_funcall(weak_obj_cache, rb_intern("[]"), 1, key_rb);
+  return ret;
+}
+
+static void WeakObjectCache_Add(const void* key, VALUE val) {
+  PBRUBY_ASSERT(WeakObjectCache_Get(key) == Qnil);
+  VALUE key_rb = ObjectCache_GetKey(key);
+  rb_funcall(weak_obj_cache, rb_intern("[]="), 2, key_rb, val);
+  PBRUBY_ASSERT(WeakObjectCache_Get(key) == val);
+}
+
+#endif
+
+// Public ObjectCache API.
+
+static void ObjectCache_Init() {
+  StrongObjectCache_Init();
+#if USE_WEAK_MAP
+  WeakObjectCache_Init();
+#endif
+}
+
+void ObjectCache_Add(const void* key, VALUE val, upb_arena *arena) {
+#if USE_WEAK_MAP
+  (void)arena;
+  WeakObjectCache_Add(key, val);
+#else
+  StrongObjectCache_Add(key, val, arena);
+#endif
+}
+
+// Returns the cached object for this key, if any. Otherwise returns Qnil.
+VALUE ObjectCache_Get(const void* key) {
+#if USE_WEAK_MAP
+  return WeakObjectCache_Get(key);
+#else
+  return StrongObjectCache_Get(key);
+#endif
+}
+
+void ObjectCache_Pin(const void* key, VALUE val, upb_arena *arena) {
+#if USE_WEAK_MAP
+  PBRUBY_ASSERT(WeakObjectCache_Get(key) == val);
+  // This will GC-root the object, but we'll still use the weak map for
+  // actual lookup.
+  StrongObjectCache_Add(key, val, arena);
+#else
+  // Value is already pinned, nothing to do.
+#endif
+}
+
+/*
+ * call-seq:
+ *     Google::Protobuf.discard_unknown(msg)
+ *
+ * Discard unknown fields in the given message object and recursively discard
+ * unknown fields in submessages.
+ */
+static VALUE Google_Protobuf_discard_unknown(VALUE self, VALUE msg_rb) {
+  const upb_msgdef *m;
+  upb_msg *msg = Message_GetMutable(msg_rb, &m);
+  if (!upb_msg_discardunknown(msg, m, 128)) {
+    rb_raise(rb_eRuntimeError, "Messages nested too deeply.");
+  }
+
+  return Qnil;
+}
+
+/*
+ * call-seq:
+ *     Google::Protobuf.deep_copy(obj) => copy_of_obj
+ *
+ * Performs a deep copy of a RepeatedField instance, a Map instance, or a
+ * message object, recursively copying its members.
+ */
+VALUE Google_Protobuf_deep_copy(VALUE self, VALUE obj) {
+  VALUE klass = CLASS_OF(obj);
+  if (klass == cRepeatedField) {
+    return RepeatedField_deep_copy(obj);
+  } else if (klass == cMap) {
+    return Map_deep_copy(obj);
+  } else {
+    VALUE new_arena_rb = Arena_new();
+    upb_arena *new_arena = Arena_get(new_arena_rb);
+    const upb_msgdef *m;
+    const upb_msg *msg = Message_Get(obj, &m);
+    upb_msg* new_msg = Message_deep_copy(msg, m, new_arena);
+    return Message_GetRubyWrapper(new_msg, m, new_arena_rb);
+  }
+}
 
 // -----------------------------------------------------------------------------
 // Initialization/entry point.
@@ -83,39 +373,24 @@ ID descriptor_instancevar_interned;
 
 // This must be named "Init_protobuf_c" because the Ruby module is named
 // "protobuf_c" -- the VM looks for this symbol in our .so.
+__attribute__ ((visibility ("default")))
 void Init_protobuf_c() {
+  ObjectCache_Init();
+
   VALUE google = rb_define_module("Google");
   VALUE protobuf = rb_define_module_under(google, "Protobuf");
-  VALUE internal = rb_define_module_under(protobuf, "Internal");
-
-  descriptor_instancevar_interned = rb_intern(kDescriptorInstanceVar);
-  DescriptorPool_register(protobuf);
-  Descriptor_register(protobuf);
-  FileDescriptor_register(protobuf);
-  FieldDescriptor_register(protobuf);
-  OneofDescriptor_register(protobuf);
-  EnumDescriptor_register(protobuf);
-  MessageBuilderContext_register(internal);
-  OneofBuilderContext_register(internal);
-  EnumBuilderContext_register(internal);
-  FileBuilderContext_register(internal);
-  Builder_register(internal);
+
+  Arena_register(protobuf);
+  Defs_register(protobuf);
   RepeatedField_register(protobuf);
   Map_register(protobuf);
+  Message_register(protobuf);
 
   cError = rb_const_get(protobuf, rb_intern("Error"));
-  cParseError = rb_const_get(protobuf, rb_intern("ParseError"));
   cTypeError = rb_const_get(protobuf, rb_intern("TypeError"));
 
   rb_define_singleton_method(protobuf, "discard_unknown",
                              Google_Protobuf_discard_unknown, 1);
   rb_define_singleton_method(protobuf, "deep_copy",
                              Google_Protobuf_deep_copy, 1);
-
-  kRubyStringUtf8Encoding = rb_utf8_encoding();
-  kRubyStringASCIIEncoding = rb_usascii_encoding();
-  kRubyString8bitEncoding = rb_ascii8bit_encoding();
-
-  rb_gc_register_address(&upb_def_to_ruby_obj_map);
-  upb_def_to_ruby_obj_map = rb_hash_new();
 }