grpc 1.47.0 → 1.48.0.pre1

data/src/core/lib/channel/channel_fwd.h

@@ -0,0 +1,26 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_CHANNEL_CHANNEL_FWD_H
+#define GRPC_CORE_LIB_CHANNEL_CHANNEL_FWD_H
+
+typedef struct grpc_channel_stack grpc_channel_stack;
+typedef struct grpc_channel_filter grpc_channel_filter;
+
+typedef struct grpc_channel_element grpc_channel_element;
+typedef struct grpc_call_element grpc_call_element;
+
+typedef struct grpc_call_stack grpc_call_stack;
+
+#endif  // GRPC_CORE_LIB_CHANNEL_CHANNEL_FWD_H

data/src/core/lib/channel/channel_stack.cc

@@ -143,8 +143,8 @@ grpc_error_handle grpc_channel_stack_init(
     elems[i].channel_data = user_data;
     grpc_error_handle error =
         elems[i].filter->init_channel_elem(&elems[i], &args);
-    if (error != GRPC_ERROR_NONE) {
-      if (first_error == GRPC_ERROR_NONE) {
+    if (!GRPC_ERROR_IS_NONE(error)) {
+      if (GRPC_ERROR_IS_NONE(first_error)) {
         first_error = error;
       } else {
         GRPC_ERROR_UNREF(error);
@@ -205,8 +205,8 @@ grpc_error_handle grpc_call_stack_init(
   for (size_t i = 0; i < count; i++) {
     grpc_error_handle error =
         call_elems[i].filter->init_call_elem(&call_elems[i], elem_args);
-    if (error != GRPC_ERROR_NONE) {
-      if (first_error == GRPC_ERROR_NONE) {
+    if (!GRPC_ERROR_IS_NONE(error)) {
+      if (GRPC_ERROR_IS_NONE(first_error)) {
         first_error = error;
       } else {
         GRPC_ERROR_UNREF(error);

data/src/core/lib/channel/channel_stack.h

@@ -56,6 +56,7 @@
 #include <grpc/status.h>
 #include <grpc/support/log.h>
 
+#include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/context.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gpr/time_precise.h"
@@ -70,17 +71,6 @@
 #include "src/core/lib/resource_quota/arena.h"
 #include "src/core/lib/transport/transport.h"
 
-struct grpc_call_element;
-struct grpc_call_stack;
-struct grpc_channel_element;
-struct grpc_channel_stack;
-
-typedef struct grpc_channel_element grpc_channel_element;
-typedef struct grpc_call_element grpc_call_element;
-
-typedef struct grpc_channel_stack grpc_channel_stack;
-typedef struct grpc_call_stack grpc_call_stack;
-
 struct grpc_channel_element_args {
   grpc_channel_stack* channel_stack;
   const grpc_channel_args* channel_args;

data/src/core/lib/channel/channel_stack_builder.h

@@ -26,13 +26,10 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/surface/channel_stack_type.h"
-
-typedef struct grpc_channel_stack grpc_channel_stack;
-typedef struct grpc_channel_element grpc_channel_element;
-typedef struct grpc_channel_filter grpc_channel_filter;
-typedef struct grpc_transport grpc_transport;
+#include "src/core/lib/transport/transport_fwd.h"
 
 namespace grpc_core {
 

data/src/core/lib/channel/channel_stack_builder_impl.cc

@@ -77,7 +77,7 @@ ChannelStackBuilderImpl::Build() {
       channel_stack);
   grpc_channel_args_destroy(c_args);
 
-  if (error != GRPC_ERROR_NONE) {
+  if (!GRPC_ERROR_IS_NONE(error)) {
     grpc_channel_stack_destroy(channel_stack);
     gpr_free(channel_stack);
     auto status = grpc_error_to_absl_status(error);

data/src/core/lib/channel/channel_stack_builder_impl.h

@@ -19,6 +19,7 @@
 
 #include "absl/status/statusor.h"
 
+#include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/channel_stack_builder.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 

data/src/core/lib/channel/channelz.cc

@@ -24,6 +24,7 @@
 
 #include <algorithm>
 #include <atomic>
+#include <memory>
 #include <type_traits>
 
 #include "absl/status/statusor.h"
@@ -437,7 +438,7 @@ void PopulateSocketAddressJson(Json::Object* json, const char* name,
     grpc_resolved_address resolved_host;
     grpc_error_handle error =
         grpc_string_to_sockaddr(&resolved_host, host.c_str(), port_num);
-    if (error == GRPC_ERROR_NONE) {
+    if (GRPC_ERROR_IS_NONE(error)) {
       std::string packed_host = grpc_sockaddr_get_packed_host(&resolved_host);
       std::string b64_host = absl::Base64Escape(packed_host);
       data["tcpip_address"] = Json::Object{

data/src/core/lib/channel/channelz.h

@@ -29,8 +29,8 @@
 #include <set>
 #include <string>
 #include <utility>
+#include <vector>
 
-#include "absl/container/inlined_vector.h"
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 
@@ -170,8 +170,7 @@ class CallCountingHelper {
   // collects the sharded data into one CounterData struct.
   void CollectData(CounterData* out);
 
-  // Really zero-sized, but 0-sized arrays are illegal on MSVC.
-  absl::InlinedVector<AtomicCounterData, 1> per_cpu_counter_data_storage_;
+  std::vector<AtomicCounterData> per_cpu_counter_data_storage_;
   size_t num_cores_ = 0;
 };
 

data/src/core/lib/channel/channelz_registry.cc

@@ -24,8 +24,7 @@
 #include <cstdint>
 #include <cstring>
 #include <utility>
-
-#include "absl/container/inlined_vector.h"
+#include <vector>
 
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
@@ -77,7 +76,7 @@ RefCountedPtr<BaseNode> ChannelzRegistry::InternalGet(intptr_t uuid) {
 
 std::string ChannelzRegistry::InternalGetTopChannels(
     intptr_t start_channel_id) {
-  absl::InlinedVector<RefCountedPtr<BaseNode>, 10> top_level_channels;
+  std::vector<RefCountedPtr<BaseNode>> top_level_channels;
   RefCountedPtr<BaseNode> node_after_pagination_limit;
   {
     MutexLock lock(&mu_);
@@ -116,7 +115,7 @@ std::string ChannelzRegistry::InternalGetTopChannels(
 }
 
 std::string ChannelzRegistry::InternalGetServers(intptr_t start_server_id) {
-  absl::InlinedVector<RefCountedPtr<BaseNode>, 10> servers;
+  std::vector<RefCountedPtr<BaseNode>> servers;
   RefCountedPtr<BaseNode> node_after_pagination_limit;
   {
     MutexLock lock(&mu_);
@@ -155,7 +154,7 @@ std::string ChannelzRegistry::InternalGetServers(intptr_t start_server_id) {
 }
 
 void ChannelzRegistry::InternalLogAllEntities() {
-  absl::InlinedVector<RefCountedPtr<BaseNode>, 10> nodes;
+  std::vector<RefCountedPtr<BaseNode>> nodes;
   {
     MutexLock lock(&mu_);
     for (auto& p : node_map_) {

data/src/core/lib/channel/connected_channel.cc

@@ -32,6 +32,7 @@
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/transport/transport.h"
+#include "src/core/lib/transport/transport_fwd.h"
 
 #define MAX_BUFFER_LENGTH 8192
 

data/src/core/lib/channel/connected_channel.h

@@ -21,6 +21,7 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/channel_stack_builder.h"
 #include "src/core/lib/transport/transport.h"

data/src/core/lib/channel/promise_based_filter.cc

@@ -587,11 +587,17 @@ void ClientCallData::StartBatch(grpc_transport_stream_op_batch* b) {
       recv_trailing_state_ = RecvTrailingState::kForwarded;
       HookRecvTrailingMetadata(batch);
     }
-  } else if (cancelled_error_ != GRPC_ERROR_NONE) {
+  } else if (!GRPC_ERROR_IS_NONE(cancelled_error_)) {
     batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
   }
 
-  if (batch.is_captured()) batch.ResumeWith(&flusher);
+  if (batch.is_captured()) {
+    if (!is_last()) {
+      batch.ResumeWith(&flusher);
+    } else {
+      batch.CancelWith(GRPC_ERROR_CANCELLED, &flusher);
+    }
+  }
 }
 
 // Handle cancellation.
@@ -692,7 +698,7 @@ void ClientCallData::RecvInitialMetadataReady(grpc_error_handle error) {
       abort();  // unreachable
   }
   Flusher flusher(this);
-  if (error != GRPC_ERROR_NONE) {
+  if (!GRPC_ERROR_IS_NONE(error)) {
     recv_initial_metadata_->state = RecvInitialMetadata::kResponded;
     flusher.AddClosure(
         absl::exchange(recv_initial_metadata_->original_on_ready, nullptr),
@@ -829,7 +835,7 @@ void ClientCallData::RecvTrailingMetadataReady(grpc_error_handle error) {
   }
   // If there was an error, we'll put that into the trailing metadata and
   // proceed as if there was not.
-  if (error != GRPC_ERROR_NONE) {
+  if (!GRPC_ERROR_IS_NONE(error)) {
     SetStatusFromError(recv_trailing_metadata_, error);
   }
   // Record that we've got the callback.
@@ -1149,7 +1155,7 @@ void ServerCallData::RecvInitialMetadataReady(grpc_error_handle error) {
   Flusher flusher(this);
   GPR_ASSERT(recv_initial_state_ == RecvInitialState::kForwarded);
   // If there was an error we just propagate that through
-  if (error != GRPC_ERROR_NONE) {
+  if (!GRPC_ERROR_IS_NONE(error)) {
     recv_initial_state_ = RecvInitialState::kResponded;
     flusher.AddClosure(
         absl::exchange(original_recv_initial_metadata_ready_, nullptr),

data/src/core/lib/channel/promise_based_filter.h

@@ -37,6 +37,7 @@
 
 #include "src/core/lib/channel/call_finalization.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/context.h"
 #include "src/core/lib/gprpp/debug_location.h"
@@ -170,6 +171,7 @@ class BaseCallData : public Activity, private Wakeable {
     ~Flusher();
 
     void Resume(grpc_transport_stream_op_batch* batch) {
+      GPR_ASSERT(!call_->is_last());
       release_.push_back(batch);
     }
 

data/src/core/lib/compression/compression.cc

@@ -18,12 +18,17 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <stdlib.h>
+#include <stdint.h>
 #include <string.h>
 
+#include "absl/types/optional.h"
+
 #include <grpc/compression.h>
+#include <grpc/impl/codegen/compression_types.h>
+#include <grpc/slice.h>
 
 #include "src/core/lib/compression/compression_internal.h"
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/api_trace.h"

data/src/core/lib/compression/compression_internal.cc

@@ -23,17 +23,14 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include <cstdint>
-
 #include "absl/container/inlined_vector.h"
-#include "absl/strings/str_join.h"
+#include "absl/strings/ascii.h"
 #include "absl/strings/str_split.h"
 
-#include <grpc/compression.h>
+#include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/gpr/useful.h"
-#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/surface/api_trace.h"
 
 namespace grpc_core {

data/src/core/lib/compression/compression_internal.h

@@ -21,14 +21,15 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <stdint.h>
+
 #include <initializer_list>
 
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 
-#include <grpc/compression.h>
+#include <grpc/impl/codegen/compression_types.h>
 #include <grpc/impl/codegen/grpc_types.h>
-#include <grpc/slice.h>
 
 #include "src/core/lib/gprpp/bitset.h"
 #include "src/core/lib/slice/slice.h"

data/src/core/lib/compression/message_compress.cc

@@ -22,12 +22,14 @@
 
 #include <string.h>
 
+#include <zconf.h>
 #include <zlib.h>
 
+#include <grpc/slice_buffer.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/slice/slice_refcount.h"
 
 #define OUTPUT_BLOCK_SIZE 1024
 

data/src/core/lib/compression/message_compress.h

@@ -21,9 +21,8 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <grpc/slice_buffer.h>
-
-#include "src/core/lib/compression/compression_internal.h"
+#include <grpc/impl/codegen/compression_types.h>
+#include <grpc/slice.h>
 
 /* compress 'input' to 'output' using 'algorithm'.
    On success, appends compressed slices to output and returns 1.

data/src/core/lib/debug/stats.cc

@@ -23,28 +23,28 @@
 #include <inttypes.h>
 #include <string.h>
 
+#include <algorithm>
 #include <vector>
 
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 
 #include <grpc/support/alloc.h>
-#include <grpc/support/string_util.h>
-
-#include "src/core/lib/gpr/string.h"
-#include "src/core/lib/gpr/useful.h"
+#include <grpc/support/cpu.h>
+#include <grpc/support/sync.h>
 
 grpc_stats_data* grpc_stats_per_cpu_storage = nullptr;
 static size_t g_num_cores;
+static gpr_once g_once = GPR_ONCE_INIT;
 
 void grpc_stats_init(void) {
-  g_num_cores = std::max(1u, gpr_cpu_num_cores());
-  grpc_stats_per_cpu_storage = static_cast<grpc_stats_data*>(
-      gpr_zalloc(sizeof(grpc_stats_data) * g_num_cores));
+  gpr_once_init(&g_once, []() {
+    g_num_cores = gpr_cpu_num_cores();
+    grpc_stats_per_cpu_storage = static_cast<grpc_stats_data*>(
+        gpr_zalloc(sizeof(grpc_stats_data) * g_num_cores));
+  });
 }
 
-void grpc_stats_shutdown(void) { gpr_free(grpc_stats_per_cpu_storage); }
-
 void grpc_stats_collect(grpc_stats_data* output) {
   memset(output, 0, sizeof(*output));
   for (size_t core = 0; core < g_num_cores; core++) {

data/src/core/lib/debug/stats.h

@@ -21,6 +21,8 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <stddef.h>
+
 #include <string>
 
 #include <grpc/support/atm.h>
@@ -54,7 +56,6 @@ extern grpc_stats_data* grpc_stats_per_cpu_storage;
 #endif /* defined(GRPC_COLLECT_STATS) || !defined(NDEBUG) */
 
 void grpc_stats_init(void);
-void grpc_stats_shutdown(void);
 void grpc_stats_collect(grpc_stats_data* output);
 // c = b-a
 void grpc_stats_diff(const grpc_stats_data* b, const grpc_stats_data* a,

data/src/core/lib/debug/stats_data.cc

@@ -22,9 +22,10 @@
 
 #include "src/core/lib/debug/stats_data.h"
 
+#include <inttypes.h>
+
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/gpr/useful.h"
-#include "src/core/lib/iomgr/exec_ctx.h"
 
 const char* grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {
     "client_calls_created",

data/src/core/lib/debug/stats_data.h

@@ -23,10 +23,6 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <inttypes.h>
-
-#include "src/core/lib/iomgr/exec_ctx.h"
-
 typedef enum {
   GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED,
   GRPC_STATS_COUNTER_SERVER_CALLS_CREATED,

data/src/core/lib/debug/trace.h

@@ -19,13 +19,20 @@
 #ifndef GRPC_CORE_LIB_DEBUG_TRACE_H
 #define GRPC_CORE_LIB_DEBUG_TRACE_H
 
-#include <grpc/support/port_platform.h>
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+#define GRPC_THREADSAFE_TRACER
+#endif
+#endif
 
-#include <stdbool.h>
+#include <grpc/support/port_platform.h>
 
-#include <grpc/support/atm.h>
+#ifdef GRPC_THREADSAFE_TRACER
+#include <atomic>
+#endif
 
 #include "src/core/lib/gprpp/global_config.h"
+#include "src/core/lib/gprpp/memory.h"
 
 GPR_GLOBAL_CONFIG_DECLARE_STRING(grpc_trace);
 
@@ -36,12 +43,6 @@ void grpc_tracer_init(const char* env_var_name);
 void grpc_tracer_init();
 void grpc_tracer_shutdown(void);
 
-#if defined(__has_feature)
-#if __has_feature(thread_sanitizer)
-#define GRPC_THREADSAFE_TRACER
-#endif
-#endif
-
 namespace grpc_core {
 
 class TraceFlag;
@@ -80,7 +81,7 @@ class TraceFlag {
 #if defined(GRPC_USE_TRACERS) || !defined(NDEBUG)
   bool enabled() {
 #ifdef GRPC_THREADSAFE_TRACER
-    return gpr_atm_no_barrier_load(&value_) != 0;
+    return value_.load(std::memory_order_relaxed);
 #else
     return value_;
 #endif  // GRPC_THREADSAFE_TRACER
@@ -95,7 +96,7 @@ class TraceFlag {
 
   void set_enabled(bool enabled) {
 #ifdef GRPC_THREADSAFE_TRACER
-    gpr_atm_no_barrier_store(&value_, enabled);
+    value_.store(enabled, std::memory_order_relaxed);
 #else
     value_ = enabled;
 #endif
@@ -104,7 +105,7 @@ class TraceFlag {
   TraceFlag* next_tracer_;
   const char* const name_;
 #ifdef GRPC_THREADSAFE_TRACER
-  gpr_atm value_;
+  std::atomic<bool> value_;
 #else
   bool value_;
 #endif

data/src/core/lib/event_engine/default_event_engine_factory.cc

@@ -20,7 +20,7 @@
 #include <grpc/event_engine/event_engine.h>
 
 #include "src/core/lib/event_engine/event_engine_factory.h"
-#include "src/core/lib/event_engine/iomgr_engine.h"
+#include "src/core/lib/event_engine/iomgr_engine/iomgr_engine.h"
 
 namespace grpc_event_engine {
 namespace experimental {

data/src/core/lib/event_engine/event_engine.cc

@@ -13,49 +13,54 @@
 // limitations under the License.
 #include <grpc/support/port_platform.h>
 
+#include <atomic>
 #include <functional>
 #include <memory>
+#include <utility>
 
 #include <grpc/event_engine/event_engine.h>
 
-#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/event_engine/event_engine_factory.h"
-#include "src/core/lib/event_engine/trace.h"
-#include "src/core/lib/gprpp/sync.h"
 
 namespace grpc_event_engine {
 namespace experimental {
 
 namespace {
-const std::function<std::unique_ptr<EventEngine>()>* g_event_engine_factory =
-    nullptr;
-grpc_core::Mutex* g_mu = new grpc_core::Mutex();
+std::atomic<const std::function<std::unique_ptr<EventEngine>()>*>
+    g_event_engine_factory{nullptr};
+std::atomic<EventEngine*> g_event_engine{nullptr};
 }  // namespace
 
 void SetDefaultEventEngineFactory(
-    const std::function<std::unique_ptr<EventEngine>()>* factory) {
-  grpc_core::MutexLock lock(g_mu);
-  g_event_engine_factory = factory;
+    std::function<std::unique_ptr<EventEngine>()> factory) {
+  delete g_event_engine_factory.exchange(
+      new std::function<std::unique_ptr<EventEngine>()>(std::move(factory)));
 }
 
 std::unique_ptr<EventEngine> CreateEventEngine() {
-  grpc_core::MutexLock lock(g_mu);
-  if (g_event_engine_factory != nullptr) {
-    return (*g_event_engine_factory)();
+  if (auto* factory = g_event_engine_factory.load()) {
+    return (*factory)();
   }
   return DefaultEventEngineFactory();
 }
 
 EventEngine* GetDefaultEventEngine() {
-  static EventEngine* default_event_engine = CreateEventEngine().release();
-  return default_event_engine;
+  EventEngine* engine = g_event_engine.load(std::memory_order_acquire);
+  if (engine == nullptr) {
+    auto* created = CreateEventEngine().release();
+    if (g_event_engine.compare_exchange_strong(engine, created,
+                                               std::memory_order_acq_rel,
+                                               std::memory_order_acquire)) {
+      engine = created;
+    } else {
+      delete created;
+    }
+  }
+  return engine;
 }
 
-void InitializeEventEngine() {
-  GetDefaultEventEngine()->Run([]() {
-    GRPC_EVENT_ENGINE_TRACE("EventEngine:%p initialized",
-                            GetDefaultEventEngine());
-  });
+void ResetDefaultEventEngine() {
+  delete g_event_engine.exchange(nullptr, std::memory_order_acq_rel);
 }
 
 }  // namespace experimental

data/src/core/lib/event_engine/event_engine_factory.h

@@ -32,8 +32,8 @@ EventEngine* GetDefaultEventEngine();
 /// Create an EventEngine using the default factory provided at link time.
 std::unique_ptr<EventEngine> DefaultEventEngineFactory();
 
-// TODO(hork): remove this when any other EE usage is landed
-void InitializeEventEngine();
+/// Reset the default event engine
+void ResetDefaultEventEngine();
 
 }  // namespace experimental
 }  // namespace grpc_event_engine