grpc 1.47.0-x86_64-linux → 1.48.0.pre1-x86_64-linux

data/src/core/lib/surface/call.cc

@@ -25,7 +25,6 @@
 
 #include <algorithm>
 #include <atomic>
-#include <memory>
 #include <new>
 #include <string>
 #include <utility>
@@ -51,7 +50,6 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/channel/channel_stack_builder.h"
 #include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/channel/context.h"
 #include "src/core/lib/compression/compression_internal.h"
@@ -60,8 +58,6 @@
 #include "src/core/lib/gpr/time_precise.h"
 #include "src/core/lib/gprpp/cpp_impl_of.h"
 #include "src/core/lib/gprpp/debug_location.h"
-#include "src/core/lib/gprpp/manual_constructor.h"
-#include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/iomgr/call_combiner.h"
@@ -69,6 +65,7 @@
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/resource_quota/arena.h"
+#include "src/core/lib/slice/slice_buffer.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_refcount.h"
 #include "src/core/lib/surface/api_trace.h"
@@ -77,7 +74,6 @@
 #include "src/core/lib/surface/completion_queue.h"
 #include "src/core/lib/surface/server.h"
 #include "src/core/lib/surface/validate_metadata.h"
-#include "src/core/lib/transport/byte_stream.h"
 #include "src/core/lib/transport/error_utils.h"
 #include "src/core/lib/transport/metadata_batch.h"
 #include "src/core/lib/transport/transport.h"
@@ -105,6 +101,7 @@ class Call : public CppImplOf<Call, grpc_call> {
                                      bool is_notify_tag_closure) = 0;
   virtual bool failed_before_recv_message() const = 0;
   virtual bool is_trailers_only() const = 0;
+  virtual absl::string_view GetServerAuthority() const = 0;
   virtual void ExternalRef() = 0;
   virtual void ExternalUnref() = 0;
   virtual void InternalRef(const char* reason) = 0;
@@ -235,6 +232,13 @@ class FilterStackCall final : public Call {
     return call_failed_before_recv_message_;
   }
 
+  absl::string_view GetServerAuthority() const override {
+    const Slice* authority_metadata =
+        recv_initial_metadata_.get_pointer(HttpAuthorityMetadata());
+    if (authority_metadata == nullptr) return "";
+    return authority_metadata->as_string_view();
+  }
+
   grpc_compression_algorithm test_only_compression_algorithm() override {
     return incoming_compression_algorithm_;
   }
@@ -302,8 +306,6 @@ class FilterStackCall final : public Call {
 
     void PostCompletion();
     void FinishStep();
-    void ContinueReceivingSlices();
-    void ReceivingSliceReady(grpc_error_handle error);
     void ProcessDataAfterMetadata();
     void ReceivingStreamReady(grpc_error_handle error);
     void ValidateFilteredMetadata();
@@ -360,7 +362,6 @@ class FilterStackCall final : public Call {
   bool received_initial_metadata_ = false;
   bool receiving_message_ = false;
   bool requested_final_op_ = false;
-  gpr_atm any_ops_sent_atm_ = 0;
   gpr_atm received_final_op_atm_ = 0;
 
   BatchControl* active_batches_[kMaxConcurrentBatches] = {};
@@ -393,13 +394,13 @@ class FilterStackCall final : public Call {
   /* Contexts for various subsystems (security, tracing, ...). */
   grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
 
-  ManualConstructor<SliceBufferByteStream> sending_stream_;
+  SliceBuffer send_slice_buffer_;
+  absl::optional<SliceBuffer> receiving_slice_buffer_;
+  uint32_t receiving_stream_flags_;
 
-  OrphanablePtr<ByteStream> receiving_stream_;
   bool call_failed_before_recv_message_ = false;
   grpc_byte_buffer** receiving_buffer_ = nullptr;
   grpc_slice receiving_slice_ = grpc_empty_slice();
-  grpc_closure receiving_slice_ready_;
   grpc_closure receiving_stream_ready_;
   grpc_closure receiving_initial_metadata_ready_;
   grpc_closure receiving_trailing_metadata_ready_;
@@ -521,8 +522,8 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
 
   auto add_init_error = [](grpc_error_handle* composite,
                            grpc_error_handle new_err) {
-    if (new_err == GRPC_ERROR_NONE) return;
-    if (*composite == GRPC_ERROR_NONE) {
+    if (GRPC_ERROR_IS_NONE(new_err)) return;
+    if (GRPC_ERROR_IS_NONE(*composite)) {
       *composite = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Call creation failed");
     }
     *composite = grpc_error_add_child(*composite, new_err);
@@ -582,7 +583,7 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
     call->PublishToParent(parent);
   }
 
-  if (error != GRPC_ERROR_NONE) {
+  if (!GRPC_ERROR_IS_NONE(error)) {
     call->CancelWithError(GRPC_ERROR_REF(error));
   }
   if (args->cq != nullptr) {
@@ -646,7 +647,7 @@ void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
   auto* c = static_cast<FilterStackCall*>(call);
   c->recv_initial_metadata_.Clear();
   c->recv_trailing_metadata_.Clear();
-  c->receiving_stream_.reset();
+  c->receiving_slice_buffer_.reset();
   ParentCall* pc = c->parent_call();
   if (pc != nullptr) {
     pc->~ParentCall();
@@ -700,8 +701,7 @@ void FilterStackCall::ExternalUnref() {
 
   GPR_ASSERT(!destroy_called_);
   destroy_called_ = true;
-  bool cancel = gpr_atm_acq_load(&any_ops_sent_atm_) != 0 &&
-                gpr_atm_acq_load(&received_final_op_atm_) == 0;
+  bool cancel = gpr_atm_acq_load(&received_final_op_atm_) == 0;
   if (cancel) {
     CancelWithError(GRPC_ERROR_CANCELLED);
   } else {
@@ -818,7 +818,7 @@ void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
     }
   } else {
     *final_op_.server.cancelled =
-        error != GRPC_ERROR_NONE || !sent_server_trailing_metadata_;
+        !GRPC_ERROR_IS_NONE(error) || !sent_server_trailing_metadata_;
     channelz::ServerNode* channelz_node =
         final_op_.server.core_server->channelz_node();
     if (channelz_node != nullptr) {
@@ -952,7 +952,7 @@ void FilterStackCall::RecvInitialFilter(grpc_metadata_batch* b) {
 
 void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
                                          grpc_error_handle batch_error) {
-  if (batch_error != GRPC_ERROR_NONE) {
+  if (!GRPC_ERROR_IS_NONE(batch_error)) {
     SetFinalStatus(batch_error);
   } else {
     absl::optional<grpc_status_code> grpc_status =
@@ -972,7 +972,7 @@ void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
       if (grpc_message.has_value()) {
         error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE,
                                    grpc_message->as_string_view());
-      } else if (error != GRPC_ERROR_NONE) {
+      } else if (!GRPC_ERROR_IS_NONE(error)) {
         error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "");
       }
       SetFinalStatus(GRPC_ERROR_REF(error));
@@ -1077,12 +1077,13 @@ void FilterStackCall::BatchControl::PostCompletion() {
   }
   if (op_.send_message) {
     if (op_.payload->send_message.stream_write_closed &&
-        error == GRPC_ERROR_NONE) {
+        GRPC_ERROR_IS_NONE(error)) {
       error = grpc_error_add_child(
           error, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                      "Attempt to send message after stream was closed."));
     }
     call->sending_message_ = false;
+    call->send_slice_buffer_.Clear();
   }
   if (op_.send_trailing_metadata) {
     call->send_trailing_metadata_.Clear();
@@ -1094,7 +1095,7 @@ void FilterStackCall::BatchControl::PostCompletion() {
     GRPC_ERROR_UNREF(error);
     error = GRPC_ERROR_NONE;
   }
-  if (error != GRPC_ERROR_NONE && op_.recv_message &&
+  if (!GRPC_ERROR_IS_NONE(error) && op_.recv_message &&
       *call->receiving_buffer_ != nullptr) {
     grpc_byte_buffer_destroy(*call->receiving_buffer_);
     *call->receiving_buffer_ = nullptr;
@@ -1128,103 +1129,35 @@ void FilterStackCall::BatchControl::FinishStep() {
   }
 }
 
-void FilterStackCall::BatchControl::ContinueReceivingSlices() {
-  grpc_error_handle error;
-  FilterStackCall* call = call_;
-  for (;;) {
-    size_t remaining = call->receiving_stream_->length() -
-                       (*call->receiving_buffer_)->data.raw.slice_buffer.length;
-    if (remaining == 0) {
-      call->receiving_message_ = false;
-      call->receiving_stream_.reset();
-      FinishStep();
-      return;
-    }
-    if (call->receiving_stream_->Next(remaining,
-                                      &call->receiving_slice_ready_)) {
-      error = call->receiving_stream_->Pull(&call->receiving_slice_);
-      if (error == GRPC_ERROR_NONE) {
-        grpc_slice_buffer_add(
-            &(*call->receiving_buffer_)->data.raw.slice_buffer,
-            call->receiving_slice_);
-      } else {
-        call->receiving_stream_.reset();
-        grpc_byte_buffer_destroy(*call->receiving_buffer_);
-        *call->receiving_buffer_ = nullptr;
-        call->receiving_message_ = false;
-        FinishStep();
-        GRPC_ERROR_UNREF(error);
-        return;
-      }
-    } else {
-      return;
-    }
-  }
-}
-
-void FilterStackCall::BatchControl::ReceivingSliceReady(
-    grpc_error_handle error) {
-  FilterStackCall* call = call_;
-  bool release_error = false;
-
-  if (error == GRPC_ERROR_NONE) {
-    grpc_slice slice;
-    error = call->receiving_stream_->Pull(&slice);
-    if (error == GRPC_ERROR_NONE) {
-      grpc_slice_buffer_add(&(*call->receiving_buffer_)->data.raw.slice_buffer,
-                            slice);
-      ContinueReceivingSlices();
-    } else {
-      /* Error returned by ByteStream::Pull() needs to be released manually */
-      release_error = true;
-    }
-  }
-
-  if (error != GRPC_ERROR_NONE) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures)) {
-      GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error));
-    }
-    call->receiving_stream_.reset();
-    grpc_byte_buffer_destroy(*call->receiving_buffer_);
-    *call->receiving_buffer_ = nullptr;
-    call->receiving_message_ = false;
-    FinishStep();
-    if (release_error) {
-      GRPC_ERROR_UNREF(error);
-    }
-  }
-}
-
 void FilterStackCall::BatchControl::ProcessDataAfterMetadata() {
   FilterStackCall* call = call_;
-  if (call->receiving_stream_ == nullptr) {
+  if (!call->receiving_slice_buffer_.has_value()) {
     *call->receiving_buffer_ = nullptr;
     call->receiving_message_ = false;
     FinishStep();
   } else {
-    call->test_only_last_message_flags_ = call->receiving_stream_->flags();
-    if ((call->receiving_stream_->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
+    call->test_only_last_message_flags_ = call->receiving_stream_flags_;
+    if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) &&
         (call->incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) {
       *call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create(
           nullptr, 0, call->incoming_compression_algorithm_);
     } else {
       *call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0);
     }
-    GRPC_CLOSURE_INIT(
-        &call->receiving_slice_ready_,
-        [](void* bctl, grpc_error_handle error) {
-          static_cast<BatchControl*>(bctl)->ReceivingSliceReady(error);
-        },
-        this, grpc_schedule_on_exec_ctx);
-    ContinueReceivingSlices();
+    grpc_slice_buffer_move_into(
+        call->receiving_slice_buffer_->c_slice_buffer(),
+        &(*call->receiving_buffer_)->data.raw.slice_buffer);
+    call->receiving_message_ = false;
+    call->receiving_slice_buffer_.reset();
+    FinishStep();
   }
 }
 
 void FilterStackCall::BatchControl::ReceivingStreamReady(
     grpc_error_handle error) {
   FilterStackCall* call = call_;
-  if (error != GRPC_ERROR_NONE) {
-    call->receiving_stream_.reset();
+  if (!GRPC_ERROR_IS_NONE(error)) {
+    call->receiving_slice_buffer_.reset();
     if (batch_error_.ok()) {
       batch_error_.set(error);
     }
@@ -1233,7 +1166,8 @@ void FilterStackCall::BatchControl::ReceivingStreamReady(
   /* If recv_state is kRecvNone, we will save the batch_control
    * object with rel_cas, and will not use it after the cas. Its corresponding
    * acq_load is in receiving_initial_metadata_ready() */
-  if (error != GRPC_ERROR_NONE || call->receiving_stream_ == nullptr ||
+  if (!GRPC_ERROR_IS_NONE(error) ||
+      !call->receiving_slice_buffer_.has_value() ||
       !gpr_atm_rel_cas(&call->recv_state_, kRecvNone,
                        reinterpret_cast<gpr_atm>(this))) {
     ProcessDataAfterMetadata();
@@ -1290,7 +1224,7 @@ void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
 
   GRPC_CALL_COMBINER_STOP(call->call_combiner(), "recv_initial_metadata_ready");
 
-  if (error == GRPC_ERROR_NONE) {
+  if (GRPC_ERROR_IS_NONE(error)) {
     grpc_metadata_batch* md = &call->recv_initial_metadata_;
     call->RecvInitialFilter(md);
 
@@ -1357,7 +1291,7 @@ void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) {
   if (batch_error_.ok()) {
     batch_error_.set(error);
   }
-  if (error != GRPC_ERROR_NONE) {
+  if (!GRPC_ERROR_IS_NONE(error)) {
     call_->CancelWithError(GRPC_ERROR_REF(error));
   }
   FinishStep();
@@ -1514,10 +1448,12 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
         }
         stream_op->send_message = true;
         sending_message_ = true;
-        sending_stream_.Init(
-            &op->data.send_message.send_message->data.raw.slice_buffer, flags);
-        stream_op_payload->send_message.send_message.reset(
-            sending_stream_.get());
+        send_slice_buffer_.Clear();
+        grpc_slice_buffer_move_into(
+            &op->data.send_message.send_message->data.raw.slice_buffer,
+            send_slice_buffer_.c_slice_buffer());
+        stream_op_payload->send_message.flags = flags;
+        stream_op_payload->send_message.send_message = &send_slice_buffer_;
         has_send_ops = true;
         break;
       }
@@ -1585,7 +1521,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
               GrpcMessageMetadata(),
               Slice(grpc_slice_copy(
                   *op->data.send_status_from_server.status_details)));
-          if (status_error != GRPC_ERROR_NONE) {
+          if (!GRPC_ERROR_IS_NONE(status_error)) {
             status_error = grpc_error_set_str(
                 status_error, GRPC_ERROR_STR_GRPC_MESSAGE,
                 StringViewFromSlice(
@@ -1654,8 +1590,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
         }
         receiving_message_ = true;
         stream_op->recv_message = true;
+        receiving_slice_buffer_.reset();
         receiving_buffer_ = op->data.recv_message.recv_message;
-        stream_op_payload->recv_message.recv_message = &receiving_stream_;
+        stream_op_payload->recv_message.recv_message = &receiving_slice_buffer_;
+        receiving_stream_flags_ = 0;
+        stream_op_payload->recv_message.flags = &receiving_stream_flags_;
         stream_op_payload->recv_message.call_failed_before_recv_message =
             &call_failed_before_recv_message_;
         GRPC_CLOSURE_INIT(
@@ -1766,7 +1705,6 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
     stream_op->on_complete = &bctl->finish_batch_;
   }
 
-  gpr_atm_rel_store(&any_ops_sent_atm_, 1);
   ExecuteBatch(stream_op, &bctl->start_batch_);
 
 done:
@@ -1780,10 +1718,6 @@ done_with_error:
   }
   if (stream_op->send_message) {
     sending_message_ = false;
-    // No need to invoke call->sending_stream->Orphan() explicitly.
-    // stream_op_payload->send_message.send_message.reset() calls Deletor
-    // of call->sending_stream which in-turn invokes the Orphan() method.
-    stream_op_payload->send_message.send_message.reset();
   }
   if (stream_op->send_trailing_metadata) {
     sent_final_op_ = false;
@@ -1943,6 +1877,10 @@ int grpc_call_failed_before_recv_message(const grpc_call* c) {
   return grpc_core::Call::FromC(c)->failed_before_recv_message();
 }
 
+absl::string_view grpc_call_server_authority(const grpc_call* call) {
+  return grpc_core::Call::FromC(call)->GetServerAuthority();
+}
+
 const char* grpc_call_error_to_string(grpc_call_error error) {
   switch (error) {
     case GRPC_CALL_ERROR:

data/src/core/lib/surface/call.h

@@ -24,13 +24,14 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 
 #include <grpc/impl/codegen/compression_types.h>
 #include <grpc/impl/codegen/grpc_types.h>
 #include <grpc/support/log.h>
 
-#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/context.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
@@ -125,6 +126,9 @@ grpc_compression_algorithm grpc_call_compression_for_level(
                     Move to surface API if requested by other languages. */
 bool grpc_call_is_trailers_only(const grpc_call* call);
 
+// Returns the authority for the call, as seen on the server side.
+absl::string_view grpc_call_server_authority(const grpc_call* call);
+
 extern grpc_core::TraceFlag grpc_call_error_trace;
 extern grpc_core::TraceFlag grpc_compression_trace;
 

data/src/core/lib/surface/channel.h

@@ -40,6 +40,7 @@
 #include <grpc/slice.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/channel_stack.h"  // IWYU pragma: keep
 #include "src/core/lib/channel/channel_stack_builder.h"
 #include "src/core/lib/channel/channelz.h"
@@ -53,6 +54,7 @@
 #include "src/core/lib/resource_quota/memory_quota.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/surface/channel_stack_type.h"
+#include "src/core/lib/transport/transport_fwd.h"
 
 /** The same as grpc_channel_destroy, but doesn't create an ExecCtx, and so
  * is safe to use from within core. */

data/src/core/lib/surface/channel_ping.cc

@@ -22,8 +22,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
+#include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/channel/channel_stack_builder.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/error.h"

data/src/core/lib/surface/completion_queue.cc

@@ -29,6 +29,7 @@
 #include <utility>
 #include <vector>
 
+#include "absl/status/status.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 
@@ -695,20 +696,20 @@ static void cq_end_op_for_next(
 
   if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
       (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-       error != GRPC_ERROR_NONE)) {
+       !GRPC_ERROR_IS_NONE(error))) {
     std::string errmsg = grpc_error_std_string(error);
     GRPC_API_TRACE(
         "cq_end_op_for_next(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg.c_str(), done, done_arg, storage));
     if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-        error != GRPC_ERROR_NONE) {
+        !GRPC_ERROR_IS_NONE(error)) {
       gpr_log(GPR_INFO, "Operation failed: tag=%p, error=%s", tag,
               errmsg.c_str());
     }
   }
   cq_next_data* cqd = static_cast<cq_next_data*> DATA_FROM_CQ(cq);
-  int is_success = (error == GRPC_ERROR_NONE);
+  int is_success = (GRPC_ERROR_IS_NONE(error));
 
   storage->tag = tag;
   storage->done = done;
@@ -736,7 +737,7 @@ static void cq_end_op_for_next(
             cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), nullptr);
         gpr_mu_unlock(cq->mu);
 
-        if (kick_error != GRPC_ERROR_NONE) {
+        if (!GRPC_ERROR_IS_NONE(kick_error)) {
           gpr_log(GPR_ERROR, "Kick failed: %s",
                   grpc_error_std_string(kick_error).c_str());
           GRPC_ERROR_UNREF(kick_error);
@@ -772,18 +773,18 @@ static void cq_end_op_for_pluck(
   GPR_TIMER_SCOPE("cq_end_op_for_pluck", 0);
 
   cq_pluck_data* cqd = static_cast<cq_pluck_data*> DATA_FROM_CQ(cq);
-  int is_success = (error == GRPC_ERROR_NONE);
+  int is_success = (GRPC_ERROR_IS_NONE(error));
 
   if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
       (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-       error != GRPC_ERROR_NONE)) {
+       !GRPC_ERROR_IS_NONE(error))) {
     std::string errmsg = grpc_error_std_string(error).c_str();
     GRPC_API_TRACE(
         "cq_end_op_for_pluck(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg.c_str(), done, done_arg, storage));
     if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-        error != GRPC_ERROR_NONE) {
+        !GRPC_ERROR_IS_NONE(error)) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag,
               errmsg.c_str());
     }
@@ -819,7 +820,7 @@ static void cq_end_op_for_pluck(
     grpc_error_handle kick_error =
         cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), pluck_worker);
     gpr_mu_unlock(cq->mu);
-    if (kick_error != GRPC_ERROR_NONE) {
+    if (!GRPC_ERROR_IS_NONE(kick_error)) {
       gpr_log(GPR_ERROR, "Kick failed: %s",
               grpc_error_std_string(kick_error).c_str());
       GRPC_ERROR_UNREF(kick_error);
@@ -831,7 +832,7 @@ static void cq_end_op_for_pluck(
 
 static void functor_callback(void* arg, grpc_error_handle error) {
   auto* functor = static_cast<grpc_completion_queue_functor*>(arg);
-  functor->functor_run(functor, error == GRPC_ERROR_NONE);
+  functor->functor_run(functor, GRPC_ERROR_IS_NONE(error));
 }
 
 /* Complete an event on a completion queue of type GRPC_CQ_CALLBACK */
@@ -845,14 +846,14 @@ static void cq_end_op_for_callback(
 
   if (GRPC_TRACE_FLAG_ENABLED(grpc_api_trace) ||
       (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-       error != GRPC_ERROR_NONE)) {
+       !GRPC_ERROR_IS_NONE(error))) {
     std::string errmsg = grpc_error_std_string(error);
     GRPC_API_TRACE(
         "cq_end_op_for_callback(cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
         6, (cq, tag, errmsg.c_str(), done, done_arg, storage));
     if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures) &&
-        error != GRPC_ERROR_NONE) {
+        !GRPC_ERROR_IS_NONE(error)) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag,
               errmsg.c_str());
     }
@@ -879,7 +880,7 @@ static void cq_end_op_for_callback(
        grpc_core::ApplicationCallbackExecCtx::Available()) ||
       grpc_iomgr_is_any_background_poller_thread()) {
     grpc_core::ApplicationCallbackExecCtx::Enqueue(functor,
-                                                   (error == GRPC_ERROR_NONE));
+                                                   (GRPC_ERROR_IS_NONE(error)));
     GRPC_ERROR_UNREF(error);
     return;
   }
@@ -1056,7 +1057,7 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline,
         POLLSET_FROM_CQ(cq), nullptr, iteration_deadline);
     gpr_mu_unlock(cq->mu);
 
-    if (err != GRPC_ERROR_NONE) {
+    if (!GRPC_ERROR_IS_NONE(err)) {
       gpr_log(GPR_ERROR, "Completion queue next failed: %s",
               grpc_error_std_string(err).c_str());
       GRPC_ERROR_UNREF(err);
@@ -1301,7 +1302,7 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag,
     cq->num_polls++;
     grpc_error_handle err =
         cq->poller_vtable->work(POLLSET_FROM_CQ(cq), &worker, deadline_millis);
-    if (err != GRPC_ERROR_NONE) {
+    if (!GRPC_ERROR_IS_NONE(err)) {
       del_plucker(cq, tag, &worker);
       gpr_mu_unlock(cq->mu);
       gpr_log(GPR_ERROR, "Completion queue pluck failed: %s",

data/src/core/lib/surface/completion_queue.h

@@ -41,7 +41,8 @@ extern grpc_core::DebugOnlyTraceFlag grpc_trace_pending_tags;
 extern grpc_core::DebugOnlyTraceFlag grpc_trace_cq_refcount;
 
 typedef struct grpc_cq_completion {
-  grpc_core::ManualConstructor<grpc_core::MultiProducerSingleConsumerQueue>
+  grpc_core::ManualConstructor<
+      grpc_core::MultiProducerSingleConsumerQueue::Node>
       node;
 
   /** user supplied tag */

data/src/core/lib/surface/init.cc

@@ -186,7 +186,6 @@ void grpc_shutdown_internal_locked(void)
     grpc_iomgr_shutdown();
     gpr_timers_global_destroy();
     grpc_tracer_shutdown();
-    grpc_stats_shutdown();
     grpc_core::Fork::GlobalShutdown();
   }
   grpc_core::ApplicationCallbackExecCtx::GlobalShutdown();

data/src/core/lib/surface/lame_client.cc

@@ -33,7 +33,7 @@
 
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_args_preconditioning.h"
-#include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/promise_based_filter.h"
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/trace.h"

data/src/core/lib/surface/lame_client.h

@@ -30,7 +30,7 @@
 #include <grpc/impl/codegen/grpc_types.h>
 
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/promise_based_filter.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/iomgr/error.h"