google-protobuf 3.11.0 → 3.12.0

data/ext/google/protobuf_c/map.c

@@ -100,11 +100,11 @@ static VALUE table_key(Map* self, VALUE key,
   return key;
 }
 
-static VALUE table_key_to_ruby(Map* self, const char* buf, size_t length) {
+static VALUE table_key_to_ruby(Map* self, upb_strview key) {
   switch (self->key_type) {
     case UPB_TYPE_BYTES:
     case UPB_TYPE_STRING: {
-      VALUE ret = rb_str_new(buf, length);
+      VALUE ret = rb_str_new(key.data, key.size);
       rb_enc_associate(ret,
                        (self->key_type == UPB_TYPE_BYTES) ?
                        kRubyString8bitEncoding : kRubyStringUtf8Encoding);
@@ -116,7 +116,7 @@ static VALUE table_key_to_ruby(Map* self, const char* buf, size_t length) {
     case UPB_TYPE_INT64:
     case UPB_TYPE_UINT32:
     case UPB_TYPE_UINT64:
-      return native_slot_get(self->key_type, Qnil, buf);
+      return native_slot_get(self->key_type, Qnil, key.data);
 
     default:
       assert(false);
@@ -289,9 +289,7 @@ VALUE Map_each(VALUE _self) {
   for (upb_strtable_begin(&it, &self->table);
        !upb_strtable_done(&it);
        upb_strtable_next(&it)) {
-
-    VALUE key = table_key_to_ruby(
-        self, upb_strtable_iter_key(&it), upb_strtable_iter_keylength(&it));
+    VALUE key = table_key_to_ruby(self, upb_strtable_iter_key(&it));
 
     upb_value v = upb_strtable_iter_value(&it);
     void* mem = value_memory(&v);
@@ -319,9 +317,7 @@ VALUE Map_keys(VALUE _self) {
   for (upb_strtable_begin(&it, &self->table);
        !upb_strtable_done(&it);
        upb_strtable_next(&it)) {
-
-    VALUE key = table_key_to_ruby(
-        self, upb_strtable_iter_key(&it), upb_strtable_iter_keylength(&it));
+    VALUE key = table_key_to_ruby(self, upb_strtable_iter_key(&it));
 
     rb_ary_push(ret, key);
   }
@@ -526,17 +522,14 @@ VALUE Map_dup(VALUE _self) {
   for (upb_strtable_begin(&it, &self->table);
        !upb_strtable_done(&it);
        upb_strtable_next(&it)) {
-
+    upb_strview k = upb_strtable_iter_key(&it);
     upb_value v = upb_strtable_iter_value(&it);
     void* mem = value_memory(&v);
     upb_value dup;
     void* dup_mem = value_memory(&dup);
     native_slot_dup(self->value_type, dup_mem, mem);
 
-    if (!upb_strtable_insert2(&new_self->table,
-                              upb_strtable_iter_key(&it),
-                              upb_strtable_iter_keylength(&it),
-                              dup)) {
+    if (!upb_strtable_insert2(&new_self->table, k.data, k.size, dup)) {
       rb_raise(rb_eRuntimeError, "Error inserting value into new table");
     }
   }
@@ -554,7 +547,7 @@ VALUE Map_deep_copy(VALUE _self) {
   for (upb_strtable_begin(&it, &self->table);
        !upb_strtable_done(&it);
        upb_strtable_next(&it)) {
-
+    upb_strview k = upb_strtable_iter_key(&it);
     upb_value v = upb_strtable_iter_value(&it);
     void* mem = value_memory(&v);
     upb_value dup;
@@ -562,10 +555,7 @@ VALUE Map_deep_copy(VALUE _self) {
     native_slot_deep_copy(self->value_type, self->value_type_class, dup_mem,
                           mem);
 
-    if (!upb_strtable_insert2(&new_self->table,
-                              upb_strtable_iter_key(&it),
-                              upb_strtable_iter_keylength(&it),
-                              dup)) {
+    if (!upb_strtable_insert2(&new_self->table, k.data, k.size, dup)) {
       rb_raise(rb_eRuntimeError, "Error inserting value into new table");
     }
   }
@@ -618,16 +608,13 @@ VALUE Map_eq(VALUE _self, VALUE _other) {
   for (upb_strtable_begin(&it, &self->table);
        !upb_strtable_done(&it);
        upb_strtable_next(&it)) {
-
+    upb_strview k = upb_strtable_iter_key(&it);
     upb_value v = upb_strtable_iter_value(&it);
     void* mem = value_memory(&v);
     upb_value other_v;
     void* other_mem = value_memory(&other_v);
 
-    if (!upb_strtable_lookup2(&other->table,
-                              upb_strtable_iter_key(&it),
-                              upb_strtable_iter_keylength(&it),
-                              &other_v)) {
+    if (!upb_strtable_lookup2(&other->table, k.data, k.size, &other_v)) {
       // Not present in other map.
       return Qfalse;
     }
@@ -655,11 +642,9 @@ VALUE Map_hash(VALUE _self) {
   VALUE hash_sym = rb_intern("hash");
 
   upb_strtable_iter it;
-  for (upb_strtable_begin(&it, &self->table);
-       !upb_strtable_done(&it);
+  for (upb_strtable_begin(&it, &self->table); !upb_strtable_done(&it);
        upb_strtable_next(&it)) {
-    VALUE key = table_key_to_ruby(
-        self, upb_strtable_iter_key(&it), upb_strtable_iter_keylength(&it));
+    VALUE key = table_key_to_ruby(self, upb_strtable_iter_key(&it));
 
     upb_value v = upb_strtable_iter_value(&it);
     void* mem = value_memory(&v);
@@ -687,8 +672,7 @@ VALUE Map_to_h(VALUE _self) {
   for (upb_strtable_begin(&it, &self->table);
        !upb_strtable_done(&it);
        upb_strtable_next(&it)) {
-    VALUE key = table_key_to_ruby(
-        self, upb_strtable_iter_key(&it), upb_strtable_iter_keylength(&it));
+    VALUE key = table_key_to_ruby(self, upb_strtable_iter_key(&it));
     upb_value v = upb_strtable_iter_value(&it);
     void* mem = value_memory(&v);
     VALUE value = native_slot_get(self->value_type,
@@ -720,11 +704,9 @@ VALUE Map_inspect(VALUE _self) {
   VALUE inspect_sym = rb_intern("inspect");
 
   upb_strtable_iter it;
-  for (upb_strtable_begin(&it, &self->table);
-       !upb_strtable_done(&it);
+  for (upb_strtable_begin(&it, &self->table); !upb_strtable_done(&it);
        upb_strtable_next(&it)) {
-    VALUE key = table_key_to_ruby(
-        self, upb_strtable_iter_key(&it), upb_strtable_iter_keylength(&it));
+    VALUE key = table_key_to_ruby(self, upb_strtable_iter_key(&it));
 
     upb_value v = upb_strtable_iter_value(&it);
     void* mem = value_memory(&v);
@@ -785,20 +767,15 @@ VALUE Map_merge_into_self(VALUE _self, VALUE hashmap) {
     for (upb_strtable_begin(&it, &other->table);
          !upb_strtable_done(&it);
          upb_strtable_next(&it)) {
+      upb_strview k = upb_strtable_iter_key(&it);
 
       // Replace any existing value by issuing a 'remove' operation first.
       upb_value v;
       upb_value oldv;
-      upb_strtable_remove2(&self->table,
-                           upb_strtable_iter_key(&it),
-                           upb_strtable_iter_keylength(&it),
-                           &oldv);
+      upb_strtable_remove2(&self->table, k.data, k.size, &oldv);
 
       v = upb_strtable_iter_value(&it);
-      upb_strtable_insert2(&self->table,
-                           upb_strtable_iter_key(&it),
-                           upb_strtable_iter_keylength(&it),
-                           v);
+      upb_strtable_insert2(&self->table, k.data, k.size, v);
     }
   } else {
     rb_raise(rb_eArgError, "Unknown type merging into Map");
@@ -822,10 +799,7 @@ bool Map_done(Map_iter* iter) {
 }
 
 VALUE Map_iter_key(Map_iter* iter) {
-  return table_key_to_ruby(
-      iter->self,
-      upb_strtable_iter_key(&iter->it),
-      upb_strtable_iter_keylength(&iter->it));
+  return table_key_to_ruby(iter->self, upb_strtable_iter_key(&iter->it));
 }
 
 VALUE Map_iter_value(Map_iter* iter) {

data/ext/google/protobuf_c/message.c

@@ -242,9 +242,14 @@ static int extract_method_call(VALUE method_name, MessageHeader* self,
   // Method calls like 'has_foo?' are not allowed if field "foo" does not have
   // a hasbit (e.g. repeated fields or non-message type fields for proto3
   // syntax).
-  if (accessor_type == METHOD_PRESENCE && test_f != NULL &&
-      !upb_fielddef_haspresence(test_f)) {
-    return METHOD_UNKNOWN;
+  if (accessor_type == METHOD_PRESENCE && test_f != NULL) {
+    if (!upb_fielddef_haspresence(test_f)) return METHOD_UNKNOWN;
+
+    // TODO(haberman): remove this case, allow for proto3 oneofs.
+    if (upb_fielddef_realcontainingoneof(test_f) &&
+        upb_filedef_syntax(upb_fielddef_file(test_f)) == UPB_SYNTAX_PROTO3) {
+      return METHOD_UNKNOWN;
+    }
   }
 
   *o = test_o;
@@ -605,11 +610,17 @@ VALUE Message_inspect(VALUE _self) {
  */
 VALUE Message_to_h(VALUE _self) {
   MessageHeader* self;
-  VALUE hash;
+  VALUE hash = rb_hash_new();
   upb_msg_field_iter it;
+  bool is_proto2;
   TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
 
-  hash = rb_hash_new();
+  // We currently have a few behaviors that are specific to proto2.
+  // This is unfortunate, we should key behaviors off field attributes (like
+  // whether a field has presence), not proto2 vs. proto3. We should see if we
+  // can change this without breaking users.
+  is_proto2 =
+      upb_msgdef_syntax(self->descriptor->msgdef) == UPB_SYNTAX_PROTO2;
 
   for (upb_msg_field_begin(&it, self->descriptor->msgdef);
        !upb_msg_field_done(&it);
@@ -618,10 +629,9 @@ VALUE Message_to_h(VALUE _self) {
     VALUE msg_value;
     VALUE msg_key;
 
-    // For proto2, do not include fields which are not set.
-    if (upb_msgdef_syntax(self->descriptor->msgdef) == UPB_SYNTAX_PROTO2 &&
-       field_contains_hasbit(self->descriptor->layout, field) &&
-       !layout_has(self->descriptor->layout, Message_data(self), field)) {
+    // Do not include fields that are not present (oneof or optional fields).
+    if (is_proto2 && upb_fielddef_haspresence(field) &&
+        !layout_has(self->descriptor->layout, Message_data(self), field)) {
       continue;
     }
 
@@ -631,8 +641,7 @@ VALUE Message_to_h(VALUE _self) {
       msg_value = Map_to_h(msg_value);
     } else if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
       msg_value = RepeatedField_to_ary(msg_value);
-      if (upb_msgdef_syntax(self->descriptor->msgdef) == UPB_SYNTAX_PROTO2 &&
-          RARRAY_LEN(msg_value) == 0) {
+      if (is_proto2 && RARRAY_LEN(msg_value) == 0) {
         continue;
       }
 

data/ext/google/protobuf_c/protobuf.h

@@ -285,6 +285,7 @@ VALUE MessageBuilderContext_initialize(VALUE _self,
                                        VALUE _file_builder,
                                        VALUE name);
 VALUE MessageBuilderContext_optional(int argc, VALUE* argv, VALUE _self);
+VALUE MessageBuilderContext_proto3_optional(int argc, VALUE* argv, VALUE _self);
 VALUE MessageBuilderContext_required(int argc, VALUE* argv, VALUE _self);
 VALUE MessageBuilderContext_repeated(int argc, VALUE* argv, VALUE _self);
 VALUE MessageBuilderContext_map(int argc, VALUE* argv, VALUE _self);

data/ext/google/protobuf_c/storage.c

@@ -496,11 +496,14 @@ void create_layout(Descriptor* desc) {
   const upb_msgdef *msgdef = desc->msgdef;
   MessageLayout* layout = ALLOC(MessageLayout);
   int nfields = upb_msgdef_numfields(msgdef);
-  int noneofs = upb_msgdef_numoneofs(msgdef);
+  int noneofs = upb_msgdef_numrealoneofs(msgdef);
   upb_msg_field_iter it;
   upb_msg_oneof_iter oit;
   size_t off = 0;
   size_t hasbit = 0;
+  int i;
+
+  (void)i;
 
   layout->empty_template = NULL;
   layout->desc = desc;
@@ -513,11 +516,22 @@ void create_layout(Descriptor* desc) {
     layout->oneofs = ALLOC_N(MessageOneof, noneofs);
   }
 
+#ifndef NDEBUG
+  for (i = 0; i < nfields; i++) {
+    layout->fields[i].offset = -1;
+  }
+
+  for (i = 0; i < noneofs; i++) {
+    layout->oneofs[i].offset = -1;
+  }
+#endif
+
   for (upb_msg_field_begin(&it, msgdef);
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
     const upb_fielddef* field = upb_msg_iter_field(&it);
-    if (upb_fielddef_haspresence(field)) {
+    if (upb_fielddef_haspresence(field) &&
+        !upb_fielddef_realcontainingoneof(field)) {
       layout->fields[upb_fielddef_index(field)].hasbit = hasbit++;
     } else {
       layout->fields[upb_fielddef_index(field)].hasbit =
@@ -540,7 +554,7 @@ void create_layout(Descriptor* desc) {
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
     const upb_fielddef* field = upb_msg_iter_field(&it);
-    if (upb_fielddef_containingoneof(field) || !upb_fielddef_isseq(field) ||
+    if (upb_fielddef_realcontainingoneof(field) || !upb_fielddef_isseq(field) ||
         upb_fielddef_ismap(field)) {
       continue;
     }
@@ -555,7 +569,7 @@ void create_layout(Descriptor* desc) {
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
     const upb_fielddef* field = upb_msg_iter_field(&it);
-    if (upb_fielddef_containingoneof(field) || !upb_fielddef_isseq(field) ||
+    if (upb_fielddef_realcontainingoneof(field) || !upb_fielddef_isseq(field) ||
         !upb_fielddef_ismap(field)) {
       continue;
     }
@@ -572,7 +586,7 @@ void create_layout(Descriptor* desc) {
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
     const upb_fielddef* field = upb_msg_iter_field(&it);
-    if (upb_fielddef_containingoneof(field) || !is_value_field(field) ||
+    if (upb_fielddef_realcontainingoneof(field) || !is_value_field(field) ||
         upb_fielddef_isseq(field)) {
       continue;
     }
@@ -589,7 +603,7 @@ void create_layout(Descriptor* desc) {
     const upb_fielddef* field = upb_msg_iter_field(&it);
     size_t field_size;
 
-    if (upb_fielddef_containingoneof(field) || is_value_field(field)) {
+    if (upb_fielddef_realcontainingoneof(field) || is_value_field(field)) {
       continue;
     }
 
@@ -624,6 +638,10 @@ void create_layout(Descriptor* desc) {
     // Always allocate NATIVE_SLOT_MAX_SIZE bytes, but share the slot between
     // all fields.
     size_t field_size = NATIVE_SLOT_MAX_SIZE;
+
+    if (upb_oneofdef_issynthetic(oneof)) continue;
+    assert(upb_oneofdef_index(oneof) < noneofs);
+
     // Align the offset.
     off = align_up_to(off, field_size);
     // Assign all fields in the oneof this same offset.
@@ -643,6 +661,8 @@ void create_layout(Descriptor* desc) {
        upb_msg_oneof_next(&oit)) {
     const upb_oneofdef* oneof = upb_msg_iter_oneof(&oit);
     size_t field_size = sizeof(uint32_t);
+    if (upb_oneofdef_issynthetic(oneof)) continue;
+    assert(upb_oneofdef_index(oneof) < noneofs);
     // Align the offset.
     off = (off + field_size - 1) & ~(field_size - 1);
     layout->oneofs[upb_oneofdef_index(oneof)].case_offset = off;
@@ -652,6 +672,16 @@ void create_layout(Descriptor* desc) {
   layout->size = off;
   layout->msgdef = msgdef;
 
+#ifndef NDEBUG
+  for (i = 0; i < nfields; i++) {
+    assert(layout->fields[i].offset != -1);
+  }
+
+  for (i = 0; i < noneofs; i++) {
+    assert(layout->oneofs[i].offset != -1);
+  }
+#endif
+
   // Create the empty message template.
   layout->empty_template = ALLOC_N(char, layout->size);
   memset(layout->empty_template, 0, layout->size);
@@ -725,10 +755,7 @@ static bool slot_is_hasbit_set(MessageLayout* layout,
                             const void* storage,
                             const upb_fielddef* field) {
   size_t hasbit = layout->fields[upb_fielddef_index(field)].hasbit;
-  if (hasbit == MESSAGE_FIELD_NO_HASBIT) {
-    return false;
-  }
-
+  assert(field_contains_hasbit(layout, field));
   return DEREF_OFFSET(
       (uint8_t*)storage, hasbit / 8, char) & (1 << (hasbit % 8));
 }
@@ -736,15 +763,21 @@ static bool slot_is_hasbit_set(MessageLayout* layout,
 VALUE layout_has(MessageLayout* layout,
                  const void* storage,
                  const upb_fielddef* field) {
-  assert(field_contains_hasbit(layout, field));
-  return slot_is_hasbit_set(layout, storage, field) ? Qtrue : Qfalse;
+  const upb_oneofdef* oneof = upb_fielddef_realcontainingoneof(field);
+  assert(upb_fielddef_haspresence(field));
+  if (oneof) {
+    uint32_t oneof_case = slot_read_oneof_case(layout, storage, oneof);
+    return oneof_case == upb_fielddef_number(field) ? Qtrue : Qfalse;
+  } else {
+    return slot_is_hasbit_set(layout, storage, field) ? Qtrue : Qfalse;
+  }
 }
 
 void layout_clear(MessageLayout* layout,
                  const void* storage,
                  const upb_fielddef* field) {
   void* memory = slot_memory(layout, storage, field);
-  const upb_oneofdef* oneof = upb_fielddef_containingoneof(field);
+  const upb_oneofdef* oneof = upb_fielddef_realcontainingoneof(field);
 
   if (field_contains_hasbit(layout, field)) {
     slot_clear_hasbit(layout, storage, field);
@@ -837,7 +870,7 @@ VALUE layout_get(MessageLayout* layout,
                  const void* storage,
                  const upb_fielddef* field) {
   void* memory = slot_memory(layout, storage, field);
-  const upb_oneofdef* oneof = upb_fielddef_containingoneof(field);
+  const upb_oneofdef* oneof = upb_fielddef_realcontainingoneof(field);
   bool field_set;
   if (field_contains_hasbit(layout, field)) {
     field_set = slot_is_hasbit_set(layout, storage, field);
@@ -910,7 +943,7 @@ void layout_set(MessageLayout* layout,
                 const upb_fielddef* field,
                 VALUE val) {
   void* memory = slot_memory(layout, storage, field);
-  const upb_oneofdef* oneof = upb_fielddef_containingoneof(field);
+  const upb_oneofdef* oneof = upb_fielddef_realcontainingoneof(field);
 
   if (oneof) {
     uint32_t* oneof_case = slot_oneof_case(layout, storage, oneof);
@@ -953,7 +986,16 @@ void layout_set(MessageLayout* layout,
 
   if (layout->fields[upb_fielddef_index(field)].hasbit !=
       MESSAGE_FIELD_NO_HASBIT) {
-    slot_set_hasbit(layout, storage, field);
+    if (val == Qnil) {
+      // No other field type has a hasbit and allows nil assignment.
+      if (upb_fielddef_type(field) != UPB_TYPE_MESSAGE) {
+        fprintf(stderr, "field: %s\n", upb_fielddef_fullname(field));
+      }
+      assert(upb_fielddef_type(field) == UPB_TYPE_MESSAGE);
+      slot_clear_hasbit(layout, storage, field);
+    } else {
+      slot_set_hasbit(layout, storage, field);
+    }
   }
 }
 
@@ -972,7 +1014,7 @@ void layout_init(MessageLayout* layout, void* storage) {
 
 void layout_mark(MessageLayout* layout, void* storage) {
   VALUE* values = (VALUE*)CHARPTR_AT(storage, layout->value_offset);
-  int noneofs = upb_msgdef_numoneofs(layout->msgdef);
+  int noneofs = upb_msgdef_numrealoneofs(layout->msgdef);
   int i;
 
   for (i = 0; i < layout->value_count; i++) {
@@ -994,7 +1036,7 @@ void layout_dup(MessageLayout* layout, void* to, void* from) {
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
     const upb_fielddef* field = upb_msg_iter_field(&it);
-    const upb_oneofdef* oneof = upb_fielddef_containingoneof(field);
+    const upb_oneofdef* oneof = upb_fielddef_realcontainingoneof(field);
 
     void* to_memory = slot_memory(layout, to, field);
     void* from_memory = slot_memory(layout, from, field);
@@ -1028,7 +1070,7 @@ void layout_deep_copy(MessageLayout* layout, void* to, void* from) {
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
     const upb_fielddef* field = upb_msg_iter_field(&it);
-    const upb_oneofdef* oneof = upb_fielddef_containingoneof(field);
+    const upb_oneofdef* oneof = upb_fielddef_realcontainingoneof(field);
 
     void* to_memory = slot_memory(layout, to, field);
     void* from_memory = slot_memory(layout, from, field);
@@ -1068,7 +1110,7 @@ VALUE layout_eq(MessageLayout* layout, void* msg1, void* msg2) {
        !upb_msg_field_done(&it);
        upb_msg_field_next(&it)) {
     const upb_fielddef* field = upb_msg_iter_field(&it);
-    const upb_oneofdef* oneof = upb_fielddef_containingoneof(field);
+    const upb_oneofdef* oneof = upb_fielddef_realcontainingoneof(field);
 
     void* msg1_memory = slot_memory(layout, msg1, field);
     void* msg2_memory = slot_memory(layout, msg2, field);
@@ -1095,9 +1137,16 @@ VALUE layout_eq(MessageLayout* layout, void* msg1, void* msg2) {
         return Qfalse;
       }
     } else {
-      if (slot_is_hasbit_set(layout, msg1, field) !=
-              slot_is_hasbit_set(layout, msg2, field) ||
-          !native_slot_eq(upb_fielddef_type(field),
+      if (field_contains_hasbit(layout, field) &&
+          slot_is_hasbit_set(layout, msg1, field) !=
+              slot_is_hasbit_set(layout, msg2, field)) {
+        // TODO(haberman): I don't think we should actually care about hasbits
+        // here: an unset default should be able to equal a set default. But we
+        // can address this later (will also have to make sure defaults are
+        // being properly set when hasbit is clear).
+        return Qfalse;
+      }
+      if (!native_slot_eq(upb_fielddef_type(field),
                           field_type_class(layout, field), msg1_memory,
                           msg2_memory)) {
         return Qfalse;