grpc 1.34.0 → 1.42.0

data/src/core/lib/transport/connectivity_state.h

@@ -21,13 +21,15 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <atomic>
+#include <map>
+#include <memory>
+
 #include "absl/status/status.h"
 
 #include <grpc/grpc.h>
 
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gprpp/atomic.h"
-#include "src/core/lib/gprpp/map.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
@@ -95,9 +97,9 @@ class AsyncConnectivityStateWatcherInterface
 // to be called).
 class ConnectivityStateTracker {
  public:
-  ConnectivityStateTracker(const char* name,
-                           grpc_connectivity_state state = GRPC_CHANNEL_IDLE,
-                           const absl::Status& status = absl::Status())
+  explicit ConnectivityStateTracker(
+      const char* name, grpc_connectivity_state state = GRPC_CHANNEL_IDLE,
+      const absl::Status& status = absl::Status())
       : name_(name), state_(state), status_(status) {}
 
   ~ConnectivityStateTracker();
@@ -129,7 +131,7 @@ class ConnectivityStateTracker {
 
  private:
   const char* name_;
-  Atomic<grpc_connectivity_state> state_;
+  std::atomic<grpc_connectivity_state> state_{grpc_connectivity_state()};
   absl::Status status_;
   // TODO(roth): Once we can use C++-14 heterogeneous lookups, this can
   // be a set instead of a map.

data/src/core/lib/transport/error_utils.cc

@@ -21,47 +21,58 @@
 #include "src/core/lib/transport/error_utils.h"
 
 #include <grpc/support/string_util.h>
+
+#include "src/core/lib/gprpp/status_helper.h"
 #include "src/core/lib/iomgr/error_internal.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/transport/status_conversion.h"
 
-static grpc_error* recursively_find_error_with_field(grpc_error* error,
-                                                     grpc_error_ints which) {
+static grpc_error_handle recursively_find_error_with_field(
+    grpc_error_handle error, grpc_error_ints which) {
   intptr_t unused;
   // If the error itself has a status code, return it.
   if (grpc_error_get_int(error, which, &unused)) {
     return error;
   }
-  if (grpc_error_is_special(error)) return nullptr;
+#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
+  std::vector<absl::Status> children = grpc_core::StatusGetChildren(error);
+  for (const absl::Status& child : children) {
+    grpc_error_handle result = recursively_find_error_with_field(child, which);
+    if (result != GRPC_ERROR_NONE) return result;
+  }
+#else
+  if (grpc_error_is_special(error)) return GRPC_ERROR_NONE;
   // Otherwise, search through its children.
   uint8_t slot = error->first_err;
   while (slot != UINT8_MAX) {
     grpc_linked_error* lerr =
         reinterpret_cast<grpc_linked_error*>(error->arena + slot);
-    grpc_error* result = recursively_find_error_with_field(lerr->err, which);
+    grpc_error_handle result =
+        recursively_find_error_with_field(lerr->err, which);
     if (result) return result;
     slot = lerr->next;
   }
-  return nullptr;
+#endif
+  return GRPC_ERROR_NONE;
 }
 
-void grpc_error_get_status(grpc_error* error, grpc_millis deadline,
-                           grpc_status_code* code, grpc_slice* slice,
+void grpc_error_get_status(grpc_error_handle error, grpc_millis deadline,
+                           grpc_status_code* code, std::string* message,
                            grpc_http2_error_code* http_error,
                            const char** error_string) {
   // Fast path: We expect no error.
   if (GPR_LIKELY(error == GRPC_ERROR_NONE)) {
     if (code != nullptr) *code = GRPC_STATUS_OK;
-    if (slice != nullptr) {
+    if (message != nullptr) {
       // Normally, we call grpc_error_get_str(
-      //   error, GRPC_ERROR_STR_GRPC_MESSAGE, slice).
+      //   error, GRPC_ERROR_STR_GRPC_MESSAGE, message).
       // We can fastpath since we know that:
       // 1) Error is null
       // 2) which == GRPC_ERROR_STR_GRPC_MESSAGE
-      // 3) The resulting slice is statically known.
-      // 4) Said resulting slice is of length 0 ("").
+      // 3) The resulting message is statically known.
+      // 4) Said resulting message is "".
       // This means 3 movs, instead of 10s of instructions and a strlen.
-      *slice = grpc_core::ExternallyManagedSlice("");
+      *message = "";
     }
     if (http_error != nullptr) {
       *http_error = GRPC_HTTP2_NO_ERROR;
@@ -71,9 +82,9 @@ void grpc_error_get_status(grpc_error* error, grpc_millis deadline,
 
   // Start with the parent error and recurse through the tree of children
   // until we find the first one that has a status code.
-  grpc_error* found_error =
+  grpc_error_handle found_error =
       recursively_find_error_with_field(error, GRPC_ERROR_INT_GRPC_STATUS);
-  if (found_error == nullptr) {
+  if (found_error == GRPC_ERROR_NONE) {
     /// If no grpc-status exists, retry through the tree to find a http2 error
     /// code
     found_error =
@@ -82,7 +93,7 @@ void grpc_error_get_status(grpc_error* error, grpc_millis deadline,
 
   // If we found an error with a status code above, use that; otherwise,
   // fall back to using the parent error.
-  if (found_error == nullptr) found_error = error;
+  if (found_error == GRPC_ERROR_NONE) found_error = error;
 
   grpc_status_code status = GRPC_STATUS_UNKNOWN;
   intptr_t integer;
@@ -92,11 +103,15 @@ void grpc_error_get_status(grpc_error* error, grpc_millis deadline,
                                 &integer)) {
     status = grpc_http2_error_to_grpc_status(
         static_cast<grpc_http2_error_code>(integer), deadline);
+  } else {
+#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
+    status = static_cast<grpc_status_code>(found_error.code());
+#endif
   }
   if (code != nullptr) *code = status;
 
   if (error_string != nullptr && status != GRPC_STATUS_OK) {
-    *error_string = gpr_strdup(grpc_error_string(error));
+    *error_string = gpr_strdup(grpc_error_std_string(error).c_str());
   }
 
   if (http_error != nullptr) {
@@ -114,33 +129,54 @@ void grpc_error_get_status(grpc_error* error, grpc_millis deadline,
 
   // If the error has a status message, use it.  Otherwise, fall back to
   // the error description.
-  if (slice != nullptr) {
-    if (!grpc_error_get_str(found_error, GRPC_ERROR_STR_GRPC_MESSAGE, slice)) {
-      if (!grpc_error_get_str(found_error, GRPC_ERROR_STR_DESCRIPTION, slice)) {
-        *slice = grpc_slice_from_static_string("unknown error");
+  if (message != nullptr) {
+    if (!grpc_error_get_str(found_error, GRPC_ERROR_STR_GRPC_MESSAGE,
+                            message)) {
+      if (!grpc_error_get_str(found_error, GRPC_ERROR_STR_DESCRIPTION,
+                              message)) {
+#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
+        *message = grpc_error_std_string(error);
+#else
+        *message = "unknown error";
+#endif
       }
     }
   }
 }
 
-absl::Status grpc_error_to_absl_status(grpc_error* error) {
+absl::Status grpc_error_to_absl_status(grpc_error_handle error) {
   grpc_status_code status;
   // TODO(yashykt): This should be updated once we decide on how to use the
   // absl::Status payload to capture all the contents of grpc_error.
-  grpc_slice message;
+  std::string message;
   grpc_error_get_status(error, GRPC_MILLIS_INF_FUTURE, &status, &message,
                         nullptr /* http_error */, nullptr /* error_string */);
-  return absl::Status(static_cast<absl::StatusCode>(status),
-                      absl::string_view(reinterpret_cast<const char*>(
-                                            GRPC_SLICE_START_PTR(message)),
-                                        GRPC_SLICE_LENGTH(message)));
+  return absl::Status(static_cast<absl::StatusCode>(status), message);
+}
+
+grpc_error_handle absl_status_to_grpc_error(absl::Status status) {
+  // Special error checks
+  if (status.ok()) {
+    return GRPC_ERROR_NONE;
+  }
+  return grpc_error_set_int(
+      GRPC_ERROR_CREATE_FROM_STRING_VIEW(status.message()),
+      GRPC_ERROR_INT_GRPC_STATUS, static_cast<grpc_status_code>(status.code()));
 }
 
-bool grpc_error_has_clear_grpc_status(grpc_error* error) {
+bool grpc_error_has_clear_grpc_status(grpc_error_handle error) {
   intptr_t unused;
   if (grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, &unused)) {
     return true;
   }
+#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
+  std::vector<absl::Status> children = grpc_core::StatusGetChildren(error);
+  for (const absl::Status& child : children) {
+    if (grpc_error_has_clear_grpc_status(child)) {
+      return true;
+    }
+  }
+#else
   uint8_t slot = error->first_err;
   while (slot != UINT8_MAX) {
     grpc_linked_error* lerr =
@@ -150,5 +186,6 @@ bool grpc_error_has_clear_grpc_status(grpc_error* error) {
     }
     slot = lerr->next;
   }
+#endif
   return false;
 }

data/src/core/lib/transport/error_utils.h

@@ -34,19 +34,25 @@
 /// be populated with the entire error string. If any of the attributes (code,
 /// msg, http_status, error_string) are unneeded, they can be passed as
 /// NULL.
-void grpc_error_get_status(grpc_error* error, grpc_millis deadline,
-                           grpc_status_code* code, grpc_slice* slice,
-                           grpc_http2_error_code* http_status,
+void grpc_error_get_status(grpc_error_handle error, grpc_millis deadline,
+                           grpc_status_code* code, std::string* message,
+                           grpc_http2_error_code* http_error,
                            const char** error_string);
 
-/// Utility Function to convert a grpc_error * \a error to an absl::Status.
-/// Does NOT consume a ref to grpc_error.
-absl::Status grpc_error_to_absl_status(grpc_error* error);
+/// Utility Function to convert a grpc_error_handle  \a error to an
+/// absl::Status. Does NOT consume a ref to grpc_error.
+absl::Status grpc_error_to_absl_status(grpc_error_handle error);
+
+/// Utility function to convert an absl::Status \a status to grpc_error. Note
+/// that this method does not return "special case" errors such as
+/// GRPC_ERROR_CANCELLED, with the exception of GRPC_ERROR_NONE returned for
+/// \a absl::OkStatus().
+grpc_error_handle absl_status_to_grpc_error(absl::Status status);
 
 /// A utility function to check whether there is a clear status code that
 /// doesn't need to be guessed in \a error. This means that \a error or some
 /// child has GRPC_ERROR_INT_GRPC_STATUS set, or that it is GRPC_ERROR_NONE or
 /// GRPC_ERROR_CANCELLED
-bool grpc_error_has_clear_grpc_status(grpc_error* error);
+bool grpc_error_has_clear_grpc_status(grpc_error_handle error);
 
 #endif /* GRPC_CORE_LIB_TRANSPORT_ERROR_UTILS_H */

data/src/core/lib/transport/metadata.cc

@@ -145,10 +145,10 @@ AllocatedMetadata::AllocatedMetadata(
 AllocatedMetadata::~AllocatedMetadata() {
   grpc_slice_unref_internal(key());
   grpc_slice_unref_internal(value());
-  void* user_data = user_data_.data.Load(grpc_core::MemoryOrder::RELAXED);
+  void* user_data = user_data_.data.load(std::memory_order_relaxed);
   if (user_data) {
     destroy_user_data_func destroy_user_data =
-        user_data_.destroy_user_data.Load(grpc_core::MemoryOrder::RELAXED);
+        user_data_.destroy_user_data.load(std::memory_order_relaxed);
     destroy_user_data(user_data);
   }
 }
@@ -189,10 +189,10 @@ InternedMetadata::InternedMetadata(const grpc_slice& key,
 InternedMetadata::~InternedMetadata() {
   grpc_slice_unref_internal(key());
   grpc_slice_unref_internal(value());
-  void* user_data = user_data_.data.Load(grpc_core::MemoryOrder::RELAXED);
+  void* user_data = user_data_.data.load(std::memory_order_relaxed);
   if (user_data) {
     destroy_user_data_func destroy_user_data =
-        user_data_.destroy_user_data.Load(grpc_core::MemoryOrder::RELAXED);
+        user_data_.destroy_user_data.load(std::memory_order_relaxed);
     destroy_user_data(user_data);
   }
 }
@@ -252,7 +252,7 @@ void grpc_mdctx_global_shutdown() {
     if (shard->count != 0) {
       gpr_log(GPR_ERROR, "WARNING: %" PRIuPTR " metadata elements were leaked",
               shard->count);
-      for (int i = 0; i < shard->capacity; i++) {
+      for (size_t i = 0; i < shard->capacity; i++) {
         for (InternedMetadata* md = shard->elems[i].next; md;
              md = md->bucket_next()) {
           char* key_str = grpc_slice_to_c_string(md->key());
@@ -279,9 +279,9 @@ void grpc_mdctx_global_shutdown() {
 #ifndef NDEBUG
 static int is_mdelem_static(grpc_mdelem e) {
   return reinterpret_cast<grpc_core::StaticMetadata*>(GRPC_MDELEM_DATA(e)) >=
-             &grpc_static_mdelem_table()[0] &&
+             &grpc_core::g_static_mdelem_table[0] &&
          reinterpret_cast<grpc_core::StaticMetadata*>(GRPC_MDELEM_DATA(e)) <
-             &grpc_static_mdelem_table()[GRPC_STATIC_MDELEM_COUNT];
+             &grpc_core::g_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
 }
 #endif
 
@@ -549,20 +549,41 @@ grpc_mdelem grpc_mdelem_from_slices(
 }
 
 grpc_mdelem grpc_mdelem_from_grpc_metadata(grpc_metadata* metadata) {
-  bool changed = false;
+  bool key_changed = false;
   grpc_slice key_slice =
-      grpc_slice_maybe_static_intern(metadata->key, &changed);
+      grpc_slice_maybe_static_intern(metadata->key, &key_changed);
+  bool value_changed = false;
+  grpc_slice* unref_slice = nullptr;
   grpc_slice value_slice =
-      grpc_slice_maybe_static_intern(metadata->value, &changed);
-  return grpc_mdelem_create(
-      key_slice, value_slice,
-      changed ? nullptr : reinterpret_cast<grpc_mdelem_data*>(metadata));
+      grpc_slice_maybe_static_intern(metadata->value, &value_changed);
+  // If key or value changed, but the other didn't.... AND the other is a NOP
+  // refcount, then we need to convert it to a slice with a refcount else we run
+  // the risk of leaving a dangling reference to that metadata on the heap via
+  // this mdelem.
+  if (key_changed && !value_changed && value_slice.refcount != nullptr &&
+      value_slice.refcount->GetType() == grpc_slice_refcount::Type::NOP) {
+    value_slice = grpc_slice_copy(value_slice);
+    unref_slice = &value_slice;
+    value_changed = true;
+  } else if (!key_changed && value_changed && key_slice.refcount != nullptr &&
+             key_slice.refcount->GetType() == grpc_slice_refcount::Type::NOP) {
+    key_slice = grpc_slice_copy(key_slice);
+    unref_slice = &key_slice;
+    key_changed = true;
+  }
+  auto mdelem =
+      grpc_mdelem_create(key_slice, value_slice,
+                         key_changed || value_changed
+                             ? nullptr
+                             : reinterpret_cast<grpc_mdelem_data*>(metadata));
+  if (unref_slice != nullptr) grpc_slice_unref_internal(*unref_slice);
+  return mdelem;
 }
 
 static void* get_user_data(UserData* user_data, void (*destroy_func)(void*)) {
-  if (user_data->destroy_user_data.Load(grpc_core::MemoryOrder::ACQUIRE) ==
+  if (user_data->destroy_user_data.load(std::memory_order_acquire) ==
       destroy_func) {
-    return user_data->data.Load(grpc_core::MemoryOrder::RELAXED);
+    return user_data->data.load(std::memory_order_relaxed);
   } else {
     return nullptr;
   }
@@ -577,7 +598,7 @@ void* grpc_mdelem_get_user_data(grpc_mdelem md, void (*destroy_func)(void*)) {
           grpc_static_mdelem_user_data
               [reinterpret_cast<grpc_core::StaticMetadata*>(
                    GRPC_MDELEM_DATA(md)) -
-               grpc_static_mdelem_table()]);
+               grpc_core::g_static_mdelem_table]);
     case GRPC_MDELEM_STORAGE_ALLOCATED: {
       auto* am = reinterpret_cast<AllocatedMetadata*>(GRPC_MDELEM_DATA(md));
       return get_user_data(am->user_data(), destroy_func);
@@ -594,16 +615,16 @@ static void* set_user_data(UserData* ud, void (*destroy_func)(void*),
                            void* data) {
   GPR_ASSERT((data == nullptr) == (destroy_func == nullptr));
   grpc_core::ReleasableMutexLock lock(&ud->mu_user_data);
-  if (ud->destroy_user_data.Load(grpc_core::MemoryOrder::RELAXED)) {
+  if (ud->destroy_user_data.load(std::memory_order_relaxed)) {
     /* user data can only be set once */
-    lock.Unlock();
+    lock.Release();
     if (destroy_func != nullptr) {
       destroy_func(data);
     }
-    return ud->data.Load(grpc_core::MemoryOrder::RELAXED);
+    return ud->data.load(std::memory_order_relaxed);
   }
-  ud->data.Store(data, grpc_core::MemoryOrder::RELAXED);
-  ud->destroy_user_data.Store(destroy_func, grpc_core::MemoryOrder::RELEASE);
+  ud->data.store(data, std::memory_order_relaxed);
+  ud->destroy_user_data.store(destroy_func, std::memory_order_release);
   return data;
 }
 
@@ -619,7 +640,7 @@ void* grpc_mdelem_set_user_data(grpc_mdelem md, void (*destroy_func)(void*),
           grpc_static_mdelem_user_data
               [reinterpret_cast<grpc_core::StaticMetadata*>(
                    GRPC_MDELEM_DATA(md)) -
-               grpc_static_mdelem_table()]);
+               grpc_core::g_static_mdelem_table]);
     case GRPC_MDELEM_STORAGE_ALLOCATED: {
       auto* am = reinterpret_cast<AllocatedMetadata*>(GRPC_MDELEM_DATA(md));
       return set_user_data(am->user_data(), destroy_func, data);
@@ -673,6 +694,10 @@ void grpc_mdelem_do_unref(grpc_mdelem gmd DEBUG_ARGS) {
 
 void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
                                 uint32_t hash DEBUG_ARGS) {
+#ifndef NDEBUG
+  (void)file;
+  (void)line;
+#endif
   switch (storage) {
     case GRPC_MDELEM_STORAGE_EXTERNAL:
     case GRPC_MDELEM_STORAGE_STATIC:

data/src/core/lib/transport/metadata.h

@@ -21,14 +21,14 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <grpc/impl/codegen/log.h>
+#include <atomic>
 
 #include <grpc/grpc.h>
+#include <grpc/impl/codegen/log.h>
 #include <grpc/slice.h>
 
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gpr/useful.h"
-#include "src/core/lib/gprpp/atomic.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/slice/slice_utils.h"
 
@@ -203,8 +203,8 @@ typedef void (*destroy_user_data_func)(void* data);
 
 struct UserData {
   Mutex mu_user_data;
-  grpc_core::Atomic<destroy_user_data_func> destroy_user_data;
-  grpc_core::Atomic<void*> data;
+  std::atomic<destroy_user_data_func> destroy_user_data{nullptr};
+  std::atomic<void*> data{nullptr};
 };
 
 class StaticMetadata {
@@ -241,7 +241,7 @@ class RefcountedMdBase {
 #ifndef NDEBUG
   void Ref(const char* file, int line) {
     grpc_mdelem_trace_ref(this, key_, value_, RefValue(), file, line);
-    const intptr_t prior = refcnt_.FetchAdd(1, MemoryOrder::RELAXED);
+    const intptr_t prior = refcnt_.fetch_add(1, std::memory_order_relaxed);
     GPR_ASSERT(prior > 0);
   }
   bool Unref(const char* file, int line) {
@@ -253,10 +253,10 @@ class RefcountedMdBase {
     /* we can assume the ref count is >= 1 as the application is calling
        this function - meaning that no adjustment to mdtab_free is necessary,
        simplifying the logic here to be just an atomic increment */
-    refcnt_.FetchAdd(1, MemoryOrder::RELAXED);
+    refcnt_.fetch_add(1, std::memory_order_relaxed);
   }
   bool Unref() {
-    const intptr_t prior = refcnt_.FetchSub(1, MemoryOrder::ACQ_REL);
+    const intptr_t prior = refcnt_.fetch_sub(1, std::memory_order_acq_rel);
     GPR_DEBUG_ASSERT(prior > 0);
     return prior == 1;
   }
@@ -266,15 +266,17 @@ class RefcountedMdBase {
   void TraceAtStart(const char* tag);
 #endif
 
-  intptr_t RefValue() { return refcnt_.Load(MemoryOrder::RELAXED); }
-  bool AllRefsDropped() { return refcnt_.Load(MemoryOrder::ACQUIRE) == 0; }
-  bool FirstRef() { return refcnt_.FetchAdd(1, MemoryOrder::RELAXED) == 0; }
+  intptr_t RefValue() { return refcnt_.load(std::memory_order_relaxed); }
+  bool AllRefsDropped() { return refcnt_.load(std::memory_order_acquire) == 0; }
+  bool FirstRef() {
+    return refcnt_.fetch_add(1, std::memory_order_relaxed) == 0;
+  }
 
  private:
   /* must be byte compatible with grpc_mdelem_data */
   grpc_slice key_;
   grpc_slice value_;
-  grpc_core::Atomic<intptr_t> refcnt_;
+  std::atomic<intptr_t> refcnt_{0};
   uint32_t hash_ = 0;
 };
 
@@ -407,7 +409,8 @@ inline void grpc_mdelem_unref(grpc_mdelem gmd) {
   (GRPC_SLICE_LENGTH(GRPC_MDKEY((e))) + GRPC_SLICE_LENGTH(GRPC_MDVALUE((e))) + \
    32)
 
-#define GRPC_MDSTR_KV_HASH(k_hash, v_hash) (GPR_ROTL((k_hash), 2) ^ (v_hash))
+#define GRPC_MDSTR_KV_HASH(k_hash, v_hash) \
+  (::grpc_core::RotateLeft(size_t(k_hash), size_t(2)) ^ (v_hash))
 
 void grpc_mdctx_global_init(void);
 void grpc_mdctx_global_shutdown();