grpc 1.63.0 → 1.65.2

data/src/core/lib/surface/completion_queue.cc

@@ -15,8 +15,6 @@
 // limitations under the License.
 //
 //
-#include <grpc/support/port_platform.h>
-
 #include "src/core/lib/surface/completion_queue.h"
 
 #include <inttypes.h>
@@ -29,6 +27,8 @@
 #include <utility>
 #include <vector>
 
+#include "absl/log/check.h"
+#include "absl/log/log.h"
 #include "absl/status/status.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
@@ -37,12 +37,10 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
 
-#include "src/core/lib/debug/stats.h"
-#include "src/core/lib/debug/stats_data.h"
-#include "src/core/lib/gpr/spinlock.h"
 #include "src/core/lib/gprpp/atomic_utils.h"
 #include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/gprpp/ref_counted.h"
@@ -55,15 +53,14 @@
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/event_string.h"
+#include "src/core/telemetry/stats.h"
+#include "src/core/telemetry/stats_data.h"
+#include "src/core/util/spinlock.h"
 
 #ifdef GPR_WINDOWS
 #include "src/core/lib/experiments/experiments.h"
 #endif
 
-grpc_core::TraceFlag grpc_trace_operation_failures(false, "op_failure");
-grpc_core::DebugOnlyTraceFlag grpc_trace_pending_tags(false, "pending_tags");
-grpc_core::DebugOnlyTraceFlag grpc_trace_cq_refcount(false, "cq_refcount");
-
 namespace {
 
 // Specifies a cq thread local cache.
@@ -179,7 +176,7 @@ grpc_error_handle non_polling_poller_kick(
 
 void non_polling_poller_shutdown(grpc_pollset* pollset, grpc_closure* closure) {
   non_polling_poller* p = reinterpret_cast<non_polling_poller*>(pollset);
-  GPR_ASSERT(closure != nullptr);
+  CHECK_NE(closure, nullptr);
   p->shutdown = closure;
   if (p->root == nullptr) {
     grpc_core::ExecCtx::Run(DEBUG_LOCATION, closure, absl::OkStatus());
@@ -257,10 +254,10 @@ class CqEventQueue {
 
 struct cq_next_data {
   ~cq_next_data() {
-    GPR_ASSERT(queue.num_items() == 0);
+    CHECK_EQ(queue.num_items(), 0);
 #ifndef NDEBUG
     if (pending_events.load(std::memory_order_acquire) != 0) {
-      gpr_log(GPR_ERROR, "Destroying CQ without draining it fully.");
+      LOG(ERROR) << "Destroying CQ without draining it fully.";
     }
 #endif
   }
@@ -287,11 +284,10 @@ struct cq_pluck_data {
   }
 
   ~cq_pluck_data() {
-    GPR_ASSERT(completed_head.next ==
-               reinterpret_cast<uintptr_t>(&completed_head));
+    CHECK(completed_head.next == reinterpret_cast<uintptr_t>(&completed_head));
 #ifndef NDEBUG
     if (pending_events.load(std::memory_order_acquire) != 0) {
-      gpr_log(GPR_ERROR, "Destroying CQ without draining it fully.");
+      LOG(ERROR) << "Destroying CQ without draining it fully.";
     }
 #endif
   }
@@ -328,7 +324,7 @@ struct cq_callback_data {
   ~cq_callback_data() {
 #ifndef NDEBUG
     if (pending_events.load(std::memory_order_acquire) != 0) {
-      gpr_log(GPR_ERROR, "Destroying CQ without draining it fully.");
+      LOG(ERROR) << "Destroying CQ without draining it fully.";
     }
 #endif
   }
@@ -442,16 +438,14 @@ static const cq_vtable g_cq_vtable[] = {
 #define POLLSET_FROM_CQ(cq) \
   ((grpc_pollset*)((cq)->vtable->data_size + (char*)DATA_FROM_CQ(cq)))
 
-grpc_core::TraceFlag grpc_cq_pluck_trace(false, "queue_pluck");
-
-#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)     \
-  do {                                                   \
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) &&       \
-        (GRPC_TRACE_FLAG_ENABLED(grpc_cq_pluck_trace) || \
-         (event)->type != GRPC_QUEUE_TIMEOUT)) {         \
-      gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq,      \
-              grpc_event_string(event).c_str());         \
-    }                                                    \
+#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event) \
+  do {                                               \
+    if (GRPC_TRACE_FLAG_ENABLED(api) &&              \
+        (GRPC_TRACE_FLAG_ENABLED(queue_pluck) ||     \
+         (event)->type != GRPC_QUEUE_TIMEOUT)) {     \
+      gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq,  \
+              grpc_event_string(event).c_str());     \
+    }                                                \
   } while (0)
 
 static void on_pollset_shutdown_done(void* arg, grpc_error_handle error);
@@ -547,7 +541,7 @@ grpc_completion_queue* grpc_completion_queue_create_internal(
 
   // One for destroy(), one for pollset_shutdown
   new (&cq->owning_refs) grpc_core::RefCount(
-      2, grpc_trace_cq_refcount.enabled() ? "completion_queue" : nullptr);
+      2, GRPC_TRACE_FLAG_ENABLED(cq_refcount) ? "completion_queue" : nullptr);
 
   poller_vtable->init(POLLSET_FROM_CQ(cq), &cq->mu);
   vtable->init(DATA_FROM_CQ(cq), shutdown_callback);
@@ -656,7 +650,7 @@ static void cq_check_tag(grpc_completion_queue* cq, void* tag, bool lock_cq) {
     gpr_mu_unlock(cq->mu);
   }
 
-  GPR_ASSERT(found);
+  CHECK(found);
 }
 #else
 static void cq_check_tag(grpc_completion_queue* /*cq*/, void* /*tag*/,
@@ -701,14 +695,14 @@ static void cq_end_op_for_next(
     grpc_completion_queue* cq, void* tag, grpc_error_handle error,
     void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
     grpc_cq_completion* storage, bool /*internal*/) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
-      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok())) {
+  if (GRPC_TRACE_FLAG_ENABLED(api) ||
+      (GRPC_TRACE_FLAG_ENABLED(op_failure) && !error.ok())) {
     std::string errmsg = grpc_core::StatusToString(error);
     GRPC_API_TRACE(
         "cq_end_op_for_next(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg.c_str(), done, done_arg, storage));
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok()) {
+    if (GRPC_TRACE_FLAG_ENABLED(op_failure) && !error.ok()) {
       gpr_log(GPR_INFO, "Operation failed: tag=%p, error=%s", tag,
               errmsg.c_str());
     }
@@ -775,14 +769,14 @@ static void cq_end_op_for_pluck(
   cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
   int is_success = (error.ok());
 
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
-      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok())) {
+  if (GRPC_TRACE_FLAG_ENABLED(api) ||
+      (GRPC_TRACE_FLAG_ENABLED(op_failure) && !error.ok())) {
     std::string errmsg = grpc_core::StatusToString(error);
     GRPC_API_TRACE(
         "cq_end_op_for_pluck(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg.c_str(), done, done_arg, storage));
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok()) {
+    if (GRPC_TRACE_FLAG_ENABLED(op_failure) && !error.ok()) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag,
               errmsg.c_str());
     }
@@ -837,14 +831,14 @@ static void cq_end_op_for_callback(
     grpc_cq_completion* storage, bool internal) {
   cq_callback_data* cqd = static_cast<cq_callback_data*> DATA_FROM_CQ(cq);
 
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
-      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok())) {
+  if (GRPC_TRACE_FLAG_ENABLED(api) ||
+      (GRPC_TRACE_FLAG_ENABLED(op_failure) && !error.ok())) {
     std::string errmsg = grpc_core::StatusToString(error);
     GRPC_API_TRACE(
         "cq_end_op_for_callback(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg.c_str(), done, done_arg, storage));
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok()) {
+    if (GRPC_TRACE_FLAG_ENABLED(op_failure) && !error.ok()) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag,
               errmsg.c_str());
     }
@@ -906,7 +900,7 @@ class ExecCtxNext : public grpc_core::ExecCtx {
         static_cast<cq_is_finished_arg*>(check_ready_to_finish_arg_);
     grpc_completion_queue* cq = a->cq;
     cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
-    GPR_ASSERT(a->stolen_completion == nullptr);
+    CHECK_EQ(a->stolen_completion, nullptr);
 
     intptr_t current_last_seen_things_queued_ever =
         cqd->things_queued_ever.load(std::memory_order_relaxed);
@@ -935,7 +929,7 @@ class ExecCtxNext : public grpc_core::ExecCtx {
 
 #ifndef NDEBUG
 static void dump_pending_tags(grpc_completion_queue* cq) {
-  if (!GRPC_TRACE_FLAG_ENABLED(grpc_trace_pending_tags)) return;
+  if (!GRPC_TRACE_FLAG_ENABLED(pending_tags)) return;
   std::vector<std::string> parts;
   parts.push_back("PENDING TAGS:");
   gpr_mu_lock(cq->mu);
@@ -943,7 +937,7 @@ static void dump_pending_tags(grpc_completion_queue* cq) {
     parts.push_back(absl::StrFormat(" %p", cq->outstanding_tags[i]));
   }
   gpr_mu_unlock(cq->mu);
-  gpr_log(GPR_DEBUG, "%s", absl::StrJoin(parts, "").c_str());
+  VLOG(2) << absl::StrJoin(parts, "");
 }
 #else
 static void dump_pending_tags(grpc_completion_queue* /*cq*/) {}
@@ -963,7 +957,7 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
       5,
       (cq, deadline.tv_sec, deadline.tv_nsec, (int)deadline.clock_type,
        reserved));
-  GPR_ASSERT(!reserved);
+  CHECK(!reserved);
 
   dump_pending_tags(cq);
 
@@ -1069,7 +1063,7 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
   GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, &ret);
   GRPC_CQ_INTERNAL_UNREF(cq, "next");
 
-  GPR_ASSERT(is_finished_arg.stolen_completion == nullptr);
+  CHECK_EQ(is_finished_arg.stolen_completion, nullptr);
 
   return ret;
 }
@@ -1083,8 +1077,8 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
 static void cq_finish_shutdown_next(grpc_completion_queue* cq) {
   cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
 
-  GPR_ASSERT(cqd->shutdown_called);
-  GPR_ASSERT(cqd->pending_events.load(std::memory_order_relaxed) == 0);
+  CHECK(cqd->shutdown_called);
+  CHECK_EQ(cqd->pending_events.load(std::memory_order_relaxed), 0);
 
   cq->poller_vtable->shutdown(POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done);
 }
@@ -1157,7 +1151,7 @@ class ExecCtxPluck : public grpc_core::ExecCtx {
     grpc_completion_queue* cq = a->cq;
     cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
 
-    GPR_ASSERT(a->stolen_completion == nullptr);
+    CHECK_EQ(a->stolen_completion, nullptr);
     gpr_atm current_last_seen_things_queued_ever =
         cqd->things_queued_ever.load(std::memory_order_relaxed);
     if (current_last_seen_things_queued_ever !=
@@ -1197,7 +1191,7 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
   grpc_pollset_worker* worker = nullptr;
   cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
 
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_cq_pluck_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(queue_pluck)) {
     GRPC_API_TRACE(
         "grpc_completion_queue_pluck("
         "cq=%p, tag=%p, "
@@ -1208,7 +1202,7 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
         (cq, tag, deadline.tv_sec, deadline.tv_nsec, (int)deadline.clock_type,
          reserved));
   }
-  GPR_ASSERT(!reserved);
+  CHECK(!reserved);
 
   dump_pending_tags(cq);
 
@@ -1299,7 +1293,7 @@ done:
   GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, &ret);
   GRPC_CQ_INTERNAL_UNREF(cq, "pluck");
 
-  GPR_ASSERT(is_finished_arg.stolen_completion == nullptr);
+  CHECK_EQ(is_finished_arg.stolen_completion, nullptr);
 
   return ret;
 }
@@ -1312,8 +1306,8 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue* cq, void* tag,
 static void cq_finish_shutdown_pluck(grpc_completion_queue* cq) {
   cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
 
-  GPR_ASSERT(cqd->shutdown_called);
-  GPR_ASSERT(!cqd->shutdown.load(std::memory_order_relaxed));
+  CHECK(cqd->shutdown_called);
+  CHECK(!cqd->shutdown.load(std::memory_order_relaxed));
   cqd->shutdown.store(true, std::memory_order_relaxed);
 
   cq->poller_vtable->shutdown(POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done);
@@ -1349,7 +1343,7 @@ static void cq_finish_shutdown_callback(grpc_completion_queue* cq) {
   cq_callback_data* cqd = static_cast<cq_callback_data*> DATA_FROM_CQ(cq);
   auto* callback = cqd->shutdown_callback;
 
-  GPR_ASSERT(cqd->shutdown_called);
+  CHECK(cqd->shutdown_called);
 
   cq->poller_vtable->shutdown(POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done);
   if (grpc_iomgr_is_any_background_poller_thread()) {

data/src/core/lib/surface/completion_queue.h

@@ -21,25 +21,16 @@
 
 // Internal API for completion queues
 
-#include <grpc/support/port_platform.h>
-
 #include <stdint.h>
 
 #include <grpc/grpc.h>
+#include <grpc/support/port_platform.h>
 
-#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/gprpp/mpscq.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/iomgr_fwd.h"
 
-// These trace flags default to 1. The corresponding lines are only traced
-// if grpc_api_trace is also truthy
-extern grpc_core::TraceFlag grpc_cq_pluck_trace;
-extern grpc_core::TraceFlag grpc_trace_operation_failures;
-extern grpc_core::DebugOnlyTraceFlag grpc_trace_pending_tags;
-extern grpc_core::DebugOnlyTraceFlag grpc_trace_cq_refcount;
-
 typedef struct grpc_cq_completion {
   grpc_core::ManualConstructor<
       grpc_core::MultiProducerSingleConsumerQueue::Node>

data/src/core/lib/surface/completion_queue_factory.cc

@@ -16,12 +16,13 @@
 //
 //
 
-#include <grpc/support/port_platform.h>
-
 #include "src/core/lib/surface/completion_queue_factory.h"
 
+#include "absl/log/check.h"
+
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/surface/completion_queue.h"
@@ -48,8 +49,8 @@ static const grpc_completion_queue_factory g_default_cq_factory = {
 
 const grpc_completion_queue_factory* grpc_completion_queue_factory_lookup(
     const grpc_completion_queue_attributes* attributes) {
-  GPR_ASSERT(attributes->version >= 1 &&
-             attributes->version <= GRPC_CQ_CURRENT_VERSION);
+  CHECK(attributes->version >= 1 &&
+        attributes->version <= GRPC_CQ_CURRENT_VERSION);
 
   // The default factory can handle version 1 of the attributes structure. We
   // may have to change this as more fields are added to the structure
@@ -62,7 +63,7 @@ const grpc_completion_queue_factory* grpc_completion_queue_factory_lookup(
 
 grpc_completion_queue* grpc_completion_queue_create_for_next(void* reserved) {
   grpc_core::ExecCtx exec_ctx;
-  GPR_ASSERT(!reserved);
+  CHECK(!reserved);
   grpc_completion_queue_attributes attr = {1, GRPC_CQ_NEXT,
                                            GRPC_CQ_DEFAULT_POLLING, nullptr};
   return g_default_cq_factory.vtable->create(&g_default_cq_factory, &attr);
@@ -70,7 +71,7 @@ grpc_completion_queue* grpc_completion_queue_create_for_next(void* reserved) {
 
 grpc_completion_queue* grpc_completion_queue_create_for_pluck(void* reserved) {
   grpc_core::ExecCtx exec_ctx;
-  GPR_ASSERT(!reserved);
+  CHECK(!reserved);
   grpc_completion_queue_attributes attr = {1, GRPC_CQ_PLUCK,
                                            GRPC_CQ_DEFAULT_POLLING, nullptr};
   return g_default_cq_factory.vtable->create(&g_default_cq_factory, &attr);
@@ -79,7 +80,7 @@ grpc_completion_queue* grpc_completion_queue_create_for_pluck(void* reserved) {
 grpc_completion_queue* grpc_completion_queue_create_for_callback(
     grpc_completion_queue_functor* shutdown_callback, void* reserved) {
   grpc_core::ExecCtx exec_ctx;
-  GPR_ASSERT(!reserved);
+  CHECK(!reserved);
   grpc_completion_queue_attributes attr = {
       2, GRPC_CQ_CALLBACK, GRPC_CQ_DEFAULT_POLLING, shutdown_callback};
   return g_default_cq_factory.vtable->create(&g_default_cq_factory, &attr);
@@ -89,6 +90,6 @@ grpc_completion_queue* grpc_completion_queue_create(
     const grpc_completion_queue_factory* factory,
     const grpc_completion_queue_attributes* attr, void* reserved) {
   grpc_core::ExecCtx exec_ctx;
-  GPR_ASSERT(!reserved);
+  CHECK(!reserved);
   return factory->vtable->create(factory, attr);
 }

data/src/core/lib/surface/completion_queue_factory.h

@@ -19,9 +19,8 @@
 #ifndef GRPC_SRC_CORE_LIB_SURFACE_COMPLETION_QUEUE_FACTORY_H
 #define GRPC_SRC_CORE_LIB_SURFACE_COMPLETION_QUEUE_FACTORY_H
 
-#include <grpc/support/port_platform.h>
-
 #include <grpc/grpc.h>
+#include <grpc/support/port_platform.h>
 
 typedef struct grpc_completion_queue_factory_vtable {
   grpc_completion_queue* (*create)(const grpc_completion_queue_factory*,

data/src/core/lib/surface/event_string.cc

@@ -16,8 +16,6 @@
 //
 //
 
-#include <grpc/support/port_platform.h>
-
 #include "src/core/lib/surface/event_string.h"
 
 #include <algorithm>
@@ -26,6 +24,8 @@
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 
+#include <grpc/support/port_platform.h>
+
 static void addhdr(grpc_event* ev, std::vector<std::string>* buf) {
   buf->push_back(absl::StrFormat("tag:%p", ev->tag));
 }

data/src/core/lib/surface/event_string.h

@@ -19,11 +19,10 @@
 #ifndef GRPC_SRC_CORE_LIB_SURFACE_EVENT_STRING_H
 #define GRPC_SRC_CORE_LIB_SURFACE_EVENT_STRING_H
 
-#include <grpc/support/port_platform.h>
-
 #include <string>
 
 #include <grpc/grpc.h>
+#include <grpc/support/port_platform.h>
 
 // Returns a string describing an event. Must be later freed with gpr_free()
 std::string grpc_event_string(grpc_event* ev);