grpc 1.50.0 → 1.56.0

data/src/core/lib/surface/completion_queue.cc

@@ -1,20 +1,20 @@
-/*
- *
- * Copyright 2015-2016 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
+//
+//
+// Copyright 2015-2016 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+//
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/surface/completion_queue.h"
@@ -24,6 +24,7 @@
 
 #include <algorithm>
 #include <atomic>
+#include <initializer_list>
 #include <new>
 #include <string>
 #include <utility>
@@ -34,18 +35,19 @@
 #include "absl/strings/str_join.h"
 
 #include <grpc/grpc.h>
-#include <grpc/impl/codegen/gpr_types.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
+#include <grpc/support/time.h>
 
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/debug/stats_data.h"
 #include "src/core/lib/gpr/spinlock.h"
-#include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gprpp/atomic_utils.h"
 #include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/status_helper.h"
 #include "src/core/lib/gprpp/time.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
@@ -66,8 +68,8 @@ namespace {
 // with a cq cache will go into that cache, and
 // will only be returned on the thread that initialized the cache.
 // NOTE: Only one event will ever be cached.
-GPR_THREAD_LOCAL(grpc_cq_completion*) g_cached_event;
-GPR_THREAD_LOCAL(grpc_completion_queue*) g_cached_cq;
+thread_local grpc_cq_completion* g_cached_event;
+thread_local grpc_completion_queue* g_cached_cq;
 
 struct plucker {
   grpc_pollset_worker** worker;
@@ -115,10 +117,10 @@ grpc_error_handle non_polling_poller_work(grpc_pollset* pollset,
                                           grpc_pollset_worker** worker,
                                           grpc_core::Timestamp deadline) {
   non_polling_poller* npp = reinterpret_cast<non_polling_poller*>(pollset);
-  if (npp->shutdown) return GRPC_ERROR_NONE;
+  if (npp->shutdown) return absl::OkStatus();
   if (npp->kicked_without_poller) {
     npp->kicked_without_poller = false;
-    return GRPC_ERROR_NONE;
+    return absl::OkStatus();
   }
   non_polling_worker w;
   gpr_cv_init(&w.cv);
@@ -140,7 +142,8 @@ grpc_error_handle non_polling_poller_work(grpc_pollset* pollset,
     npp->root = w.next;
     if (&w == npp->root) {
       if (npp->shutdown) {
-        grpc_core::ExecCtx::Run(DEBUG_LOCATION, npp->shutdown, GRPC_ERROR_NONE);
+        grpc_core::ExecCtx::Run(DEBUG_LOCATION, npp->shutdown,
+                                absl::OkStatus());
       }
       npp->root = nullptr;
     }
@@ -149,7 +152,7 @@ grpc_error_handle non_polling_poller_work(grpc_pollset* pollset,
   w.prev->next = w.next;
   gpr_cv_destroy(&w.cv);
   if (worker != nullptr) *worker = nullptr;
-  return GRPC_ERROR_NONE;
+  return absl::OkStatus();
 }
 
 grpc_error_handle non_polling_poller_kick(
@@ -168,7 +171,7 @@ grpc_error_handle non_polling_poller_kick(
   } else {
     p->kicked_without_poller = true;
   }
-  return GRPC_ERROR_NONE;
+  return absl::OkStatus();
 }
 
 void non_polling_poller_shutdown(grpc_pollset* pollset, grpc_closure* closure) {
@@ -176,7 +179,7 @@ void non_polling_poller_shutdown(grpc_pollset* pollset, grpc_closure* closure) {
   GPR_ASSERT(closure != nullptr);
   p->shutdown = closure;
   if (p->root == nullptr) {
-    grpc_core::ExecCtx::Run(DEBUG_LOCATION, closure, GRPC_ERROR_NONE);
+    grpc_core::ExecCtx::Run(DEBUG_LOCATION, closure, absl::OkStatus());
   } else {
     non_polling_worker* w = p->root;
     do {
@@ -187,13 +190,13 @@ void non_polling_poller_shutdown(grpc_pollset* pollset, grpc_closure* closure) {
 }
 
 const cq_poller_vtable g_poller_vtable_by_poller_type[] = {
-    /* GRPC_CQ_DEFAULT_POLLING */
+    // GRPC_CQ_DEFAULT_POLLING
     {true, true, grpc_pollset_size, grpc_pollset_init, grpc_pollset_kick,
      grpc_pollset_work, grpc_pollset_shutdown, grpc_pollset_destroy},
-    /* GRPC_CQ_NON_LISTENING */
+    // GRPC_CQ_NON_LISTENING
     {true, false, grpc_pollset_size, grpc_pollset_init, grpc_pollset_kick,
      grpc_pollset_work, grpc_pollset_shutdown, grpc_pollset_destroy},
-    /* GRPC_CQ_NON_POLLING */
+    // GRPC_CQ_NON_POLLING
     {false, false, non_polling_poller_size, non_polling_poller_init,
      non_polling_poller_kick, non_polling_poller_work,
      non_polling_poller_shutdown, non_polling_poller_destroy},
@@ -219,17 +222,17 @@ struct cq_vtable {
 
 namespace {
 
-/* Queue that holds the cq_completion_events. Internally uses
- * MultiProducerSingleConsumerQueue (a lockfree multiproducer single consumer
- * queue). It uses a queue_lock to support multiple consumers.
- * Only used in completion queues whose completion_type is GRPC_CQ_NEXT */
+// Queue that holds the cq_completion_events. Internally uses
+// MultiProducerSingleConsumerQueue (a lockfree multiproducer single consumer
+// queue). It uses a queue_lock to support multiple consumers.
+// Only used in completion queues whose completion_type is GRPC_CQ_NEXT
 class CqEventQueue {
  public:
   CqEventQueue() = default;
   ~CqEventQueue() = default;
 
-  /* Note: The counter is not incremented/decremented atomically with push/pop.
-   * The count is only eventually consistent */
+  // Note: The counter is not incremented/decremented atomically with push/pop.
+  // The count is only eventually consistent
   intptr_t num_items() const {
     return num_queue_items_.load(std::memory_order_relaxed);
   }
@@ -238,14 +241,14 @@ class CqEventQueue {
   grpc_cq_completion* Pop();
 
  private:
-  /* Spinlock to serialize consumers i.e pop() operations */
+  // Spinlock to serialize consumers i.e pop() operations
   gpr_spinlock queue_lock_ = GPR_SPINLOCK_INITIALIZER;
 
   grpc_core::MultiProducerSingleConsumerQueue queue_;
 
-  /* A lazy counter of number of items in the queue. This is NOT atomically
-     incremented/decremented along with push/pop operations and hence is only
-     eventually consistent */
+  // A lazy counter of number of items in the queue. This is NOT atomically
+  // incremented/decremented along with push/pop operations and hence is only
+  // eventually consistent
   std::atomic<intptr_t> num_queue_items_{0};
 };
 
@@ -259,18 +262,18 @@ struct cq_next_data {
 #endif
   }
 
-  /** Completed events for completion-queues of type GRPC_CQ_NEXT */
+  /// Completed events for completion-queues of type GRPC_CQ_NEXT
   CqEventQueue queue;
 
-  /** Counter of how many things have ever been queued on this completion queue
-      useful for avoiding locks to check the queue */
+  /// Counter of how many things have ever been queued on this completion queue
+  /// useful for avoiding locks to check the queue
   std::atomic<intptr_t> things_queued_ever{0};
 
-  /** Number of outstanding events (+1 if not shut down)
-      Initial count is dropped by grpc_completion_queue_shutdown */
+  /// Number of outstanding events (+1 if not shut down)
+  /// Initial count is dropped by grpc_completion_queue_shutdown
   std::atomic<intptr_t> pending_events{1};
 
-  /** 0 initially. 1 once we initiated shutdown */
+  /// 0 initially. 1 once we initiated shutdown
   bool shutdown_called = false;
 };
 
@@ -290,25 +293,25 @@ struct cq_pluck_data {
 #endif
   }
 
-  /** Completed events for completion-queues of type GRPC_CQ_PLUCK */
+  /// Completed events for completion-queues of type GRPC_CQ_PLUCK
   grpc_cq_completion completed_head;
   grpc_cq_completion* completed_tail;
 
-  /** Number of pending events (+1 if we're not shutdown).
-      Initial count is dropped by grpc_completion_queue_shutdown. */
+  /// Number of pending events (+1 if we're not shutdown).
+  /// Initial count is dropped by grpc_completion_queue_shutdown.
   std::atomic<intptr_t> pending_events{1};
 
-  /** Counter of how many things have ever been queued on this completion queue
-      useful for avoiding locks to check the queue */
+  /// Counter of how many things have ever been queued on this completion queue
+  /// useful for avoiding locks to check the queue
   std::atomic<intptr_t> things_queued_ever{0};
 
-  /** 0 initially. 1 once we completed shutting */
-  /* TODO: (sreek) This is not needed since (shutdown == 1) if and only if
-   * (pending_events == 0). So consider removing this in future and use
-   * pending_events */
+  /// 0 initially. 1 once we completed shutting
+  // TODO(sreek): This is not needed since (shutdown == 1) if and only if
+  // (pending_events == 0). So consider removing this in future and use
+  // pending_events
   std::atomic<bool> shutdown{false};
 
-  /** 0 initially. 1 once we initiated shutdown */
+  /// 0 initially. 1 once we initiated shutdown
   bool shutdown_called = false;
 
   int num_pluckers = 0;
@@ -327,29 +330,33 @@ struct cq_callback_data {
 #endif
   }
 
-  /** No actual completed events queue, unlike other types */
+  /// No actual completed events queue, unlike other types
 
-  /** Number of pending events (+1 if we're not shutdown).
-      Initial count is dropped by grpc_completion_queue_shutdown. */
+  /// Number of pending events (+1 if we're not shutdown).
+  /// Initial count is dropped by grpc_completion_queue_shutdown.
   std::atomic<intptr_t> pending_events{1};
 
-  /** 0 initially. 1 once we initiated shutdown */
+  /// 0 initially. 1 once we initiated shutdown
   bool shutdown_called = false;
 
-  /** A callback that gets invoked when the CQ completes shutdown */
+  /// A callback that gets invoked when the CQ completes shutdown
   grpc_completion_queue_functor* shutdown_callback;
 };
 
 }  // namespace
 
-/* Completion queue structure */
+// Completion queue structure
 struct grpc_completion_queue {
-  /** Once owning_refs drops to zero, we will destroy the cq */
+  /// Once owning_refs drops to zero, we will destroy the cq
   grpc_core::RefCount owning_refs;
-
+  /// Add the paddings to fix the false sharing
+  char padding_1[GPR_CACHELINE_SIZE];
   gpr_mu* mu;
 
+  char padding_2[GPR_CACHELINE_SIZE];
   const cq_vtable* vtable;
+
+  char padding_3[GPR_CACHELINE_SIZE];
   const cq_poller_vtable* poller_vtable;
 
 #ifndef NDEBUG
@@ -362,7 +369,7 @@ struct grpc_completion_queue {
   int num_polls;
 };
 
-/* Forward declarations */
+// Forward declarations
 static void cq_finish_shutdown_next(grpc_completion_queue* cq);
 static void cq_finish_shutdown_pluck(grpc_completion_queue* cq);
 static void cq_finish_shutdown_callback(grpc_completion_queue* cq);
@@ -412,17 +419,17 @@ static void cq_destroy_next(void* data);
 static void cq_destroy_pluck(void* data);
 static void cq_destroy_callback(void* data);
 
-/* Completion queue vtables based on the completion-type */
+// Completion queue vtables based on the completion-type
 static const cq_vtable g_cq_vtable[] = {
-    /* GRPC_CQ_NEXT */
+    // GRPC_CQ_NEXT
     {GRPC_CQ_NEXT, sizeof(cq_next_data), cq_init_next, cq_shutdown_next,
      cq_destroy_next, cq_begin_op_for_next, cq_end_op_for_next, cq_next,
      nullptr},
-    /* GRPC_CQ_PLUCK */
+    // GRPC_CQ_PLUCK
     {GRPC_CQ_PLUCK, sizeof(cq_pluck_data), cq_init_pluck, cq_shutdown_pluck,
      cq_destroy_pluck, cq_begin_op_for_pluck, cq_end_op_for_pluck, nullptr,
      cq_pluck},
-    /* GRPC_CQ_CALLBACK */
+    // GRPC_CQ_CALLBACK
     {GRPC_CQ_CALLBACK, sizeof(cq_callback_data), cq_init_callback,
      cq_shutdown_callback, cq_destroy_callback, cq_begin_op_for_callback,
      cq_end_op_for_callback, nullptr, nullptr},
@@ -460,7 +467,7 @@ int grpc_completion_queue_thread_local_cache_flush(grpc_completion_queue* cq,
   if (storage != nullptr && g_cached_cq == cq) {
     *tag = storage->tag;
     grpc_core::ExecCtx exec_ctx;
-    *ok = (storage->next & static_cast<uintptr_t>(1)) == 1;
+    *ok = (storage->next & uintptr_t{1}) == 1;
     storage->done(storage->done_arg, storage);
     ret = 1;
     cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
@@ -512,13 +519,13 @@ grpc_completion_queue* grpc_completion_queue_create_internal(
 
   switch (completion_type) {
     case GRPC_CQ_NEXT:
-      GRPC_STATS_INC_CQ_NEXT_CREATES();
+      grpc_core::global_stats().IncrementCqNextCreates();
       break;
     case GRPC_CQ_PLUCK:
-      GRPC_STATS_INC_CQ_PLUCK_CREATES();
+      grpc_core::global_stats().IncrementCqPluckCreates();
       break;
     case GRPC_CQ_CALLBACK:
-      GRPC_STATS_INC_CQ_CALLBACK_CREATES();
+      grpc_core::global_stats().IncrementCqCallbackCreates();
       break;
   }
 
@@ -535,8 +542,9 @@ grpc_completion_queue* grpc_completion_queue_create_internal(
   cq->vtable = vtable;
   cq->poller_vtable = poller_vtable;
 
-  /* One for destroy(), one for pollset_shutdown */
-  new (&cq->owning_refs) grpc_core::RefCount(2);
+  // One for destroy(), one for pollset_shutdown
+  new (&cq->owning_refs) grpc_core::RefCount(
+      2, grpc_trace_cq_refcount.enabled() ? "completion_queue" : nullptr);
 
   poller_vtable->init(POLLSET_FROM_CQ(cq), &cq->mu);
   vtable->init(DATA_FROM_CQ(cq), shutdown_callback);
@@ -683,60 +691,57 @@ bool grpc_cq_begin_op(grpc_completion_queue* cq, void* tag) {
   return cq->vtable->begin_op(cq, tag);
 }
 
-/* Queue a GRPC_OP_COMPLETED operation to a completion queue (with a
- * completion
- * type of GRPC_CQ_NEXT) */
+// Queue a GRPC_OP_COMPLETED operation to a completion queue (with a
+// completion
+// type of GRPC_CQ_NEXT)
 static void cq_end_op_for_next(
     grpc_completion_queue* cq, void* tag, grpc_error_handle error,
     void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
     grpc_cq_completion* storage, bool /*internal*/) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
-      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-       !GRPC_ERROR_IS_NONE(error))) {
-    std::string errmsg = grpc_error_std_string(error);
+      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok())) {
+    std::string errmsg = grpc_core::StatusToString(error);
     GRPC_API_TRACE(
         "cq_end_op_for_next(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg.c_str(), done, done_arg, storage));
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-        !GRPC_ERROR_IS_NONE(error)) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok()) {
       gpr_log(GPR_INFO, "Operation failed: tag=%p, error=%s", tag,
               errmsg.c_str());
     }
   }
   cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
-  int is_success = (GRPC_ERROR_IS_NONE(error));
+  int is_success = (error.ok());
 
   storage->tag = tag;
   storage->done = done;
   storage->done_arg = done_arg;
   storage->next = static_cast<uintptr_t>(is_success);
 
-  cq_check_tag(cq, tag, true); /* Used in debug builds only */
+  cq_check_tag(cq, tag, true);  // Used in debug builds only
 
   if (g_cached_cq == cq && g_cached_event == nullptr) {
     g_cached_event = storage;
   } else {
-    /* Add the completion to the queue */
+    // Add the completion to the queue
     bool is_first = cqd->queue.Push(storage);
     cqd->things_queued_ever.fetch_add(1, std::memory_order_relaxed);
-    /* Since we do not hold the cq lock here, it is important to do an 'acquire'
-       load here (instead of a 'no_barrier' load) to match with the release
-       store
-       (done via pending_events.fetch_sub(1, ACQ_REL)) in cq_shutdown_next
-       */
+    // Since we do not hold the cq lock here, it is important to do an 'acquire'
+    // load here (instead of a 'no_barrier' load) to match with the release
+    // store
+    // (done via pending_events.fetch_sub(1, ACQ_REL)) in cq_shutdown_next
+    //
     if (cqd->pending_events.load(std::memory_order_acquire) != 1) {
-      /* Only kick if this is the first item queued */
+      // Only kick if this is the first item queued
       if (is_first) {
         gpr_mu_lock(cq->mu);
         grpc_error_handle kick_error =
             cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), nullptr);
         gpr_mu_unlock(cq->mu);
 
-        if (!GRPC_ERROR_IS_NONE(kick_error)) {
+        if (!kick_error.ok()) {
           gpr_log(GPR_ERROR, "Kick failed: %s",
-                  grpc_error_std_string(kick_error).c_str());
-          GRPC_ERROR_UNREF(kick_error);
+                  grpc_core::StatusToString(kick_error).c_str());
         }
       }
       if (cqd->pending_events.fetch_sub(1, std::memory_order_acq_rel) == 1) {
@@ -755,30 +760,26 @@ static void cq_end_op_for_next(
       GRPC_CQ_INTERNAL_UNREF(cq, "shutting_down");
     }
   }
-
-  GRPC_ERROR_UNREF(error);
 }
 
-/* Queue a GRPC_OP_COMPLETED operation to a completion queue (with a
- * completion
- * type of GRPC_CQ_PLUCK) */
+// Queue a GRPC_OP_COMPLETED operation to a completion queue (with a
+// completion
+// type of GRPC_CQ_PLUCK)
 static void cq_end_op_for_pluck(
     grpc_completion_queue* cq, void* tag, grpc_error_handle error,
     void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
     grpc_cq_completion* storage, bool /*internal*/) {
   cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
-  int is_success = (GRPC_ERROR_IS_NONE(error));
+  int is_success = (error.ok());
 
   if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
-      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-       !GRPC_ERROR_IS_NONE(error))) {
-    std::string errmsg = grpc_error_std_string(error).c_str();
+      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok())) {
+    std::string errmsg = grpc_core::StatusToString(error);
     GRPC_API_TRACE(
         "cq_end_op_for_pluck(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg.c_str(), done, done_arg, storage));
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-        !GRPC_ERROR_IS_NONE(error)) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok()) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag,
               errmsg.c_str());
     }
@@ -791,9 +792,9 @@ static void cq_end_op_for_pluck(
                   static_cast<uintptr_t>(is_success);
 
   gpr_mu_lock(cq->mu);
-  cq_check_tag(cq, tag, false); /* Used in debug builds only */
+  cq_check_tag(cq, tag, false);  // Used in debug builds only
 
-  /* Add to the list of completions */
+  // Add to the list of completions
   cqd->things_queued_ever.fetch_add(1, std::memory_order_relaxed);
   cqd->completed_tail->next =
       reinterpret_cast<uintptr_t>(storage) | (1u & cqd->completed_tail->next);
@@ -814,22 +815,19 @@ static void cq_end_op_for_pluck(
     grpc_error_handle kick_error =
         cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), pluck_worker);
     gpr_mu_unlock(cq->mu);
-    if (!GRPC_ERROR_IS_NONE(kick_error)) {
+    if (!kick_error.ok()) {
       gpr_log(GPR_ERROR, "Kick failed: %s",
-              grpc_error_std_string(kick_error).c_str());
-      GRPC_ERROR_UNREF(kick_error);
+              grpc_core::StatusToString(kick_error).c_str());
     }
   }
-
-  GRPC_ERROR_UNREF(error);
 }
 
 static void functor_callback(void* arg, grpc_error_handle error) {
   auto* functor = static_cast<grpc_completion_queue_functor*>(arg);
-  functor->functor_run(functor, GRPC_ERROR_IS_NONE(error));
+  functor->functor_run(functor, error.ok());
 }
 
-/* Complete an event on a completion queue of type GRPC_CQ_CALLBACK */
+// Complete an event on a completion queue of type GRPC_CQ_CALLBACK
 static void cq_end_op_for_callback(
     grpc_completion_queue* cq, void* tag, grpc_error_handle error,
     void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg,
@@ -837,15 +835,13 @@ static void cq_end_op_for_callback(
   cq_callback_data* cqd = static_cast<cq_callback_data*> DATA_FROM_CQ(cq);
 
   if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
-      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-       !GRPC_ERROR_IS_NONE(error))) {
-    std::string errmsg = grpc_error_std_string(error);
+      (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok())) {
+    std::string errmsg = grpc_core::StatusToString(error);
     GRPC_API_TRACE(
         "cq_end_op_for_callback(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg.c_str(), done, done_arg, storage));
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-        !GRPC_ERROR_IS_NONE(error)) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) && !error.ok()) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag,
               errmsg.c_str());
     }
@@ -855,7 +851,7 @@ static void cq_end_op_for_callback(
   // for reserved storage. Invoke the done callback right away to release it.
   done(done_arg, storage);
 
-  cq_check_tag(cq, tag, true); /* Used in debug builds only */
+  cq_check_tag(cq, tag, true);  // Used in debug builds only
 
   if (cqd->pending_events.fetch_sub(1, std::memory_order_acq_rel) == 1) {
     cq_finish_shutdown_callback(cq);
@@ -871,9 +867,7 @@ static void cq_end_op_for_callback(
   if (((internal || functor->inlineable) &&
        grpc_core::ApplicationCallbackExecCtx::Available()) ||
       grpc_iomgr_is_any_background_poller_thread()) {
-    grpc_core::ApplicationCallbackExecCtx::Enqueue(functor,
-                                                   (GRPC_ERROR_IS_NONE(error)));
-    GRPC_ERROR_UNREF(error);
+    grpc_core::ApplicationCallbackExecCtx::Enqueue(functor, (error.ok()));
     return;
   }
 
@@ -896,7 +890,7 @@ struct cq_is_finished_arg {
   grpc_completion_queue* cq;
   grpc_core::Timestamp deadline;
   grpc_cq_completion* stolen_completion;
-  void* tag; /* for pluck */
+  void* tag;  // for pluck
   bool first_loop;
 };
 class ExecCtxNext : public grpc_core::ExecCtx {
@@ -919,11 +913,11 @@ class ExecCtxNext : public grpc_core::ExecCtx {
       a->last_seen_things_queued_ever =
           cqd->things_queued_ever.load(std::memory_order_relaxed);
 
-      /* Pop a cq_completion from the queue. Returns NULL if the queue is empty
-       * might return NULL in some cases even if the queue is not empty; but
-       * that
-       * is ok and doesn't affect correctness. Might effect the tail latencies a
-       * bit) */
+      // Pop a cq_completion from the queue. Returns NULL if the queue is empty
+      // might return NULL in some cases even if the queue is not empty; but
+      // that
+      // is ok and doesn't affect correctness. Might effect the tail latencies a
+      // bit)
       a->stolen_completion = cqd->queue.Pop();
       if (a->stolen_completion != nullptr) {
         return true;
@@ -1004,26 +998,26 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
       c->done(c->done_arg, c);
       break;
     } else {
-      /* If c == NULL it means either the queue is empty OR in an transient
-         inconsistent state. If it is the latter, we shold do a 0-timeout poll
-         so that the thread comes back quickly from poll to make a second
-         attempt at popping. Not doing this can potentially deadlock this
-         thread forever (if the deadline is infinity) */
+      // If c == NULL it means either the queue is empty OR in an transient
+      // inconsistent state. If it is the latter, we shold do a 0-timeout poll
+      // so that the thread comes back quickly from poll to make a second
+      // attempt at popping. Not doing this can potentially deadlock this
+      // thread forever (if the deadline is infinity)
       if (cqd->queue.num_items() > 0) {
         iteration_deadline = grpc_core::Timestamp::ProcessEpoch();
       }
     }
 
     if (cqd->pending_events.load(std::memory_order_acquire) == 0) {
-      /* Before returning, check if the queue has any items left over (since
-         MultiProducerSingleConsumerQueue::Pop() can sometimes return NULL
-         even if the queue is not empty. If so, keep retrying but do not
-         return GRPC_QUEUE_SHUTDOWN */
+      // Before returning, check if the queue has any items left over (since
+      // MultiProducerSingleConsumerQueue::Pop() can sometimes return NULL
+      // even if the queue is not empty. If so, keep retrying but do not
+      // return GRPC_QUEUE_SHUTDOWN
       if (cqd->queue.num_items() > 0) {
-        /* Go to the beginning of the loop. No point doing a poll because
-           (cq->shutdown == true) is only possible when there is no pending
-           work (i.e cq->pending_events == 0) and any outstanding completion
-           events should have already been queued on this cq */
+        // Go to the beginning of the loop. No point doing a poll because
+        // (cq->shutdown == true) is only possible when there is no pending
+        // work (i.e cq->pending_events == 0) and any outstanding completion
+        // events should have already been queued on this cq
         continue;
       }
 
@@ -1040,18 +1034,17 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
       break;
     }
 
-    /* The main polling work happens in grpc_pollset_work */
+    // The main polling work happens in grpc_pollset_work
     gpr_mu_lock(cq->mu);
     cq->num_polls++;
     grpc_error_handle err = cq->poller_vtable->work(
         POLLSET_FROM_CQ(cq), nullptr, iteration_deadline);
     gpr_mu_unlock(cq->mu);
 
-    if (!GRPC_ERROR_IS_NONE(err)) {
+    if (!err.ok()) {
       gpr_log(GPR_ERROR, "Completion queue next failed: %s",
-              grpc_error_std_string(err).c_str());
-      GRPC_ERROR_UNREF(err);
-      if (err == GRPC_ERROR_CANCELLED) {
+              grpc_core::StatusToString(err).c_str());
+      if (err == absl::CancelledError()) {
         ret.type = GRPC_QUEUE_SHUTDOWN;
       } else {
         ret.type = GRPC_QUEUE_TIMEOUT;
@@ -1078,12 +1071,12 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
   return ret;
 }
 
-/* Finishes the completion queue shutdown. This means that there are no more
-   completion events / tags expected from the completion queue
-   - Must be called under completion queue lock
-   - Must be called only once in completion queue's lifetime
-   - grpc_completion_queue_shutdown() MUST have been called before calling
-   this function */
+// Finishes the completion queue shutdown. This means that there are no more
+// completion events / tags expected from the completion queue
+// - Must be called under completion queue lock
+// - Must be called only once in completion queue's lifetime
+// - grpc_completion_queue_shutdown() MUST have been called before calling
+// this function
 static void cq_finish_shutdown_next(grpc_completion_queue* cq) {
   cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
 
@@ -1096,12 +1089,12 @@ static void cq_finish_shutdown_next(grpc_completion_queue* cq) {
 static void cq_shutdown_next(grpc_completion_queue* cq) {
   cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
 
-  /* Need an extra ref for cq here because:
-   * We call cq_finish_shutdown_next() below, that would call pollset shutdown.
-   * Pollset shutdown decrements the cq ref count which can potentially destroy
-   * the cq (if that happens to be the last ref).
-   * Creating an extra ref here prevents the cq from getting destroyed while
-   * this function is still active */
+  // Need an extra ref for cq here because:
+  // We call cq_finish_shutdown_next() below, that would call pollset shutdown.
+  // Pollset shutdown decrements the cq ref count which can potentially destroy
+  // the cq (if that happens to be the last ref).
+  // Creating an extra ref here prevents the cq from getting destroyed while
+  // this function is still active
   GRPC_CQ_INTERNAL_REF(cq, "shutting_down");
   gpr_mu_lock(cq->mu);
   if (cqd->shutdown_called) {
@@ -1110,9 +1103,9 @@ static void cq_shutdown_next(grpc_completion_queue* cq) {
     return;
   }
   cqd->shutdown_called = true;
-  /* Doing acq/release fetch_sub here to match with
-   * cq_begin_op_for_next and cq_end_op_for_next functions which read/write
-   * on this counter without necessarily holding a lock on cq */
+  // Doing acq/release fetch_sub here to match with
+  // cq_begin_op_for_next and cq_end_op_for_next functions which read/write
+  // on this counter without necessarily holding a lock on cq
   if (cqd->pending_events.fetch_sub(1, std::memory_order_acq_rel) == 1) {
     cq_finish_shutdown_next(cq);
   }
@@ -1147,7 +1140,7 @@ static void del_plucker(grpc_completion_queue* cq, void* tag,
       return;
     }
   }
-  GPR_UNREACHABLE_CODE(return );
+  GPR_UNREACHABLE_CODE(return);
 }
 
 class ExecCtxPluck : public grpc_core::ExecCtx {
@@ -1172,11 +1165,9 @@ class ExecCtxPluck : public grpc_core::ExecCtx {
       grpc_cq_completion* c;
       grpc_cq_completion* prev = &cqd->completed_head;
       while ((c = reinterpret_cast<grpc_cq_completion*>(
-                  prev->next & ~static_cast<uintptr_t>(1))) !=
-             &cqd->completed_head) {
+                  prev->next & ~uintptr_t{1})) != &cqd->completed_head) {
         if (c->tag == a->tag) {
-          prev->next = (prev->next & static_cast<uintptr_t>(1)) |
-                       (c->next & ~static_cast<uintptr_t>(1));
+          prev->next = (prev->next & uintptr_t{1}) | (c->next & ~uintptr_t{1});
           if (c == cqd->completed_tail) {
             cqd->completed_tail = prev;
           }
@@ -1243,11 +1234,9 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
     }
     prev = &cqd->completed_head;
     while ((c = reinterpret_cast<grpc_cq_completion*>(
-                prev->next & ~static_cast<uintptr_t>(1))) !=
-           &cqd->completed_head) {
-      if (c->tag == tag) {
-        prev->next = (prev->next & static_cast<uintptr_t>(1)) |
-                     (c->next & ~static_cast<uintptr_t>(1));
+                prev->next & ~uintptr_t{1})) != &cqd->completed_head) {
+      if (GPR_LIKELY(c->tag == tag)) {
+        prev->next = (prev->next & uintptr_t{1}) | (c->next & ~uintptr_t{1});
         if (c == cqd->completed_tail) {
           cqd->completed_tail = prev;
         }
@@ -1272,7 +1261,7 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
               "is %d",
               GRPC_MAX_COMPLETION_QUEUE_PLUCKERS);
       gpr_mu_unlock(cq->mu);
-      /* TODO(ctiller): should we use a different result here */
+      // TODO(ctiller): should we use a different result here
       ret.type = GRPC_QUEUE_TIMEOUT;
       ret.success = 0;
       dump_pending_tags(cq);
@@ -1290,12 +1279,11 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
     cq->num_polls++;
     grpc_error_handle err =
         cq->poller_vtable->work(POLLSET_FROM_CQ(cq), &worker, deadline_millis);
-    if (!GRPC_ERROR_IS_NONE(err)) {
+    if (!err.ok()) {
       del_plucker(cq, tag, &worker);
       gpr_mu_unlock(cq->mu);
       gpr_log(GPR_ERROR, "Completion queue pluck failed: %s",
-              grpc_error_std_string(err).c_str());
-      GRPC_ERROR_UNREF(err);
+              grpc_core::StatusToString(err).c_str());
       ret.type = GRPC_QUEUE_TIMEOUT;
       ret.success = 0;
       dump_pending_tags(cq);
@@ -1328,17 +1316,17 @@ static void cq_finish_shutdown_pluck(grpc_completion_queue* cq) {
   cq->poller_vtable->shutdown(POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done);
 }
 
-/* NOTE: This function is almost exactly identical to cq_shutdown_next() but
- * merging them is a bit tricky and probably not worth it */
+// NOTE: This function is almost exactly identical to cq_shutdown_next() but
+// merging them is a bit tricky and probably not worth it
 static void cq_shutdown_pluck(grpc_completion_queue* cq) {
   cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
 
-  /* Need an extra ref for cq here because:
-   * We call cq_finish_shutdown_pluck() below, that would call pollset shutdown.
-   * Pollset shutdown decrements the cq ref count which can potentially destroy
-   * the cq (if that happens to be the last ref).
-   * Creating an extra ref here prevents the cq from getting destroyed while
-   * this function is still active */
+  // Need an extra ref for cq here because:
+  // We call cq_finish_shutdown_pluck() below, that would call pollset shutdown.
+  // Pollset shutdown decrements the cq ref count which can potentially destroy
+  // the cq (if that happens to be the last ref).
+  // Creating an extra ref here prevents the cq from getting destroyed while
+  // this function is still active
   GRPC_CQ_INTERNAL_REF(cq, "shutting_down (pluck cq)");
   gpr_mu_lock(cq->mu);
   if (cqd->shutdown_called) {
@@ -1370,18 +1358,18 @@ static void cq_finish_shutdown_callback(grpc_completion_queue* cq) {
   // from a background poller thread.
   grpc_core::Executor::Run(
       GRPC_CLOSURE_CREATE(functor_callback, callback, nullptr),
-      GRPC_ERROR_NONE);
+      absl::OkStatus());
 }
 
 static void cq_shutdown_callback(grpc_completion_queue* cq) {
   cq_callback_data* cqd = static_cast<cq_callback_data*> DATA_FROM_CQ(cq);
 
-  /* Need an extra ref for cq here because:
-   * We call cq_finish_shutdown_callback() below, which calls pollset shutdown.
-   * Pollset shutdown decrements the cq ref count which can potentially destroy
-   * the cq (if that happens to be the last ref).
-   * Creating an extra ref here prevents the cq from getting destroyed while
-   * this function is still active */
+  // Need an extra ref for cq here because:
+  // We call cq_finish_shutdown_callback() below, which calls pollset shutdown.
+  // Pollset shutdown decrements the cq ref count which can potentially destroy
+  // the cq (if that happens to be the last ref).
+  // Creating an extra ref here prevents the cq from getting destroyed while
+  // this function is still active
   GRPC_CQ_INTERNAL_REF(cq, "shutting_down (callback cq)");
   gpr_mu_lock(cq->mu);
   if (cqd->shutdown_called) {
@@ -1399,8 +1387,8 @@ static void cq_shutdown_callback(grpc_completion_queue* cq) {
   GRPC_CQ_INTERNAL_UNREF(cq, "shutting_down (callback cq)");
 }
 
-/* Shutdown simply drops a ref that we reserved at creation time; if we drop
-   to zero here, then enter shutdown mode and wake up any waiters */
+// Shutdown simply drops a ref that we reserved at creation time; if we drop
+// to zero here, then enter shutdown mode and wake up any waiters
 void grpc_completion_queue_shutdown(grpc_completion_queue* cq) {
   grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;