grpc 1.50.0-x86_64-linux → 1.51.0-x86_64-linux

data/src/core/lib/surface/call.h

@@ -24,14 +24,17 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include "absl/functional/any_invocable.h"
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 
 #include <grpc/impl/codegen/compression_types.h>
 #include <grpc/impl/codegen/grpc_types.h>
+#include <grpc/support/atm.h>
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_fwd.h"
+#include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/context.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
@@ -39,6 +42,8 @@
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/iomgr_fwd.h"
+#include "src/core/lib/iomgr/polling_entity.h"
+#include "src/core/lib/promise/context.h"
 #include "src/core/lib/resource_quota/arena.h"
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/surface/api_trace.h"
@@ -67,6 +72,45 @@ typedef struct grpc_call_create_args {
   grpc_core::Timestamp send_deadline;
 } grpc_call_create_args;
 
+namespace grpc_core {
+class PromiseBasedCall;
+
+// TODO(ctiller): move more call things into this type
+class CallContext {
+ public:
+  explicit CallContext(PromiseBasedCall* call) : call_(call) {}
+
+  // Run some action in the call activity context. This is needed to adapt some
+  // legacy systems to promises, and will likely disappear once that conversion
+  // is complete.
+  void RunInContext(absl::AnyInvocable<void()> fn);
+
+  // TODO(ctiller): remove this once transport APIs are promise based
+  void IncrementRefCount(const char* reason = "call_context");
+
+  // TODO(ctiller): remove this once transport APIs are promise based
+  void Unref(const char* reason = "call_context");
+
+  grpc_call_stats* call_stats() { return &call_stats_; }
+  gpr_atm* peer_string_atm_ptr();
+  grpc_polling_entity* polling_entity() { return &pollent_; }
+
+ private:
+  friend class PromiseBasedCall;
+  // Call final info.
+  grpc_call_stats call_stats_;
+  // Pollset stuff, can't wait to remove.
+  // TODO(ctiller): bring forth EventEngine.
+  grpc_polling_entity pollent_;
+  // TODO(ctiller): remove this once transport APIs are promise based and we
+  // don't need refcounting here.
+  PromiseBasedCall* const call_;
+};
+
+template <>
+struct ContextType<CallContext> {};
+}  // namespace grpc_core
+
 /* Create a new call based on \a args.
    Regardless of success or failure, always returns a valid new call into *call
    */

data/src/core/lib/surface/call_details.cc

@@ -24,7 +24,7 @@
 
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/slice/slice_refcount.h"
+#include "src/core/lib/slice/slice.h"
 #include "src/core/lib/surface/api_trace.h"
 
 void grpc_call_details_init(grpc_call_details* details) {
@@ -36,6 +36,6 @@ void grpc_call_details_init(grpc_call_details* details) {
 void grpc_call_details_destroy(grpc_call_details* details) {
   GRPC_API_TRACE("grpc_call_details_destroy(details=%p)", 1, (details));
   grpc_core::ExecCtx exec_ctx;
-  grpc_slice_unref_internal(details->method);
-  grpc_slice_unref_internal(details->host);
+  grpc_core::CSliceUnref(details->method);
+  grpc_core::CSliceUnref(details->host);
 }

data/src/core/lib/surface/call_trace.cc

@@ -0,0 +1,113 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/surface/call_trace.h"
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/container/flat_hash_map.h"
+#include "absl/meta/type_traits.h"
+#include "absl/status/status.h"
+#include "absl/types/variant.h"
+
+#include <grpc/support/log.h>
+
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/gprpp/no_destruct.h"
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/arena_promise.h"
+#include "src/core/lib/transport/metadata_batch.h"
+#include "src/core/lib/transport/transport.h"
+
+namespace grpc_core {
+
+const grpc_channel_filter* PromiseTracingFilterFor(
+    const grpc_channel_filter* filter) {
+  struct DerivedFilter : public grpc_channel_filter {
+    explicit DerivedFilter(const grpc_channel_filter* filter)
+        : grpc_channel_filter{
+              /* start_transport_stream_op_batch: */ grpc_call_next_op,
+              /* make_call_promise: */
+              [](grpc_channel_element* elem, CallArgs call_args,
+                 NextPromiseFactory next_promise_factory)
+                  -> ArenaPromise<ServerMetadataHandle> {
+                auto* source_filter =
+                    static_cast<const DerivedFilter*>(elem->filter)->filter;
+                gpr_log(
+                    GPR_DEBUG,
+                    "%sCreateCallPromise[%s]: client_initial_metadata=%s",
+                    Activity::current()->DebugTag().c_str(),
+                    source_filter->name,
+                    call_args.client_initial_metadata->DebugString().c_str());
+                return [source_filter, child = next_promise_factory(
+                                           std::move(call_args))]() mutable {
+                  gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: begin",
+                          Activity::current()->DebugTag().c_str(),
+                          source_filter->name);
+                  auto r = child();
+                  if (auto* p = absl::get_if<ServerMetadataHandle>(&r)) {
+                    gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: done: %s",
+                            Activity::current()->DebugTag().c_str(),
+                            source_filter->name, (*p)->DebugString().c_str());
+                  } else {
+                    gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: <<pending>",
+                            Activity::current()->DebugTag().c_str(),
+                            source_filter->name);
+                  }
+                  return r;
+                };
+              },
+              grpc_channel_next_op, /* sizeof_call_data: */ 0,
+              /* init_call_elem: */
+              [](grpc_call_element*, const grpc_call_element_args*) {
+                return absl::OkStatus();
+              },
+              grpc_call_stack_ignore_set_pollset_or_pollset_set,
+              /* destroy_call_elem: */
+              [](grpc_call_element*, const grpc_call_final_info*,
+                 grpc_closure*) {},
+              /* sizeof_channel_data: */ 0, /* init_channel_elem: */
+              [](grpc_channel_element*, grpc_channel_element_args*) {
+                return absl::OkStatus();
+              },
+              /* post_init_channel_elem: */
+              [](grpc_channel_stack*, grpc_channel_element*) {},
+              /* destroy_channel_elem: */ [](grpc_channel_element*) {},
+              grpc_channel_next_get_info, filter->name},
+          filter(filter) {}
+    const grpc_channel_filter* const filter;
+  };
+  struct Globals {
+    Mutex mu;
+    absl::flat_hash_map<const grpc_channel_filter*,
+                        std::unique_ptr<DerivedFilter>>
+        map ABSL_GUARDED_BY(mu);
+  };
+  auto* globals = NoDestructSingleton<Globals>::Get();
+  MutexLock lock(&globals->mu);
+  auto it = globals->map.find(filter);
+  if (it != globals->map.end()) return it->second.get();
+  return globals->map.emplace(filter, std::make_unique<DerivedFilter>(filter))
+      .first->second.get();
+}
+
+}  // namespace grpc_core

data/src/core/lib/surface/call_trace.h

@@ -0,0 +1,30 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_SURFACE_CALL_TRACE_H
+#define GRPC_CORE_LIB_SURFACE_CALL_TRACE_H
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/channel/channel_fwd.h"
+#include "src/core/lib/debug/trace.h"
+
+extern grpc_core::TraceFlag grpc_call_trace;
+
+namespace grpc_core {
+const grpc_channel_filter* PromiseTracingFilterFor(
+    const grpc_channel_filter* filter);
+}
+
+#endif  // GRPC_CORE_LIB_SURFACE_CALL_TRACE_H

data/src/core/lib/surface/channel.cc

@@ -43,6 +43,7 @@
 #include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/debug/stats_data.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
@@ -51,7 +52,6 @@
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/resource_quota/memory_quota.h"
 #include "src/core/lib/resource_quota/resource_quota.h"
-#include "src/core/lib/slice/slice_refcount.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel_init.h"
@@ -63,11 +63,12 @@
 
 namespace grpc_core {
 
-Channel::Channel(bool is_client, std::string target,
+Channel::Channel(bool is_client, bool is_promising, std::string target,
                  const ChannelArgs& channel_args,
                  grpc_compression_options compression_options,
                  RefCountedPtr<grpc_channel_stack> channel_stack)
     : is_client_(is_client),
+      is_promising_(is_promising),
       compression_options_(compression_options),
       call_size_estimate_(channel_stack->call_stack_size +
                           grpc_call_get_initial_size_estimate()),
@@ -109,9 +110,9 @@ absl::StatusOr<RefCountedPtr<Channel>> Channel::CreateWithBuilder(
     ChannelStackBuilder* builder) {
   auto channel_args = builder->channel_args();
   if (builder->channel_stack_type() == GRPC_SERVER_CHANNEL) {
-    GRPC_STATS_INC_SERVER_CHANNELS_CREATED();
+    global_stats().IncrementServerChannelsCreated();
   } else {
-    GRPC_STATS_INC_CLIENT_CHANNELS_CREATED();
+    global_stats().IncrementClientChannelsCreated();
   }
   absl::StatusOr<RefCountedPtr<grpc_channel_stack>> r = builder->Build();
   if (!r.ok()) {
@@ -151,8 +152,8 @@ absl::StatusOr<RefCountedPtr<Channel>> Channel::CreateWithBuilder(
 
   return RefCountedPtr<Channel>(new Channel(
       grpc_channel_stack_type_is_client(builder->channel_stack_type()),
-      std::string(builder->target()), channel_args, compression_options,
-      std::move(*r)));
+      builder->IsPromising(), std::string(builder->target()), channel_args,
+      compression_options, std::move(*r)));
 }
 
 namespace {
@@ -169,44 +170,12 @@ void channelz_node_destroy(void* p) {
 int channelz_node_cmp(void* p1, void* p2) { return QsortCompare(p1, p2); }
 const grpc_arg_pointer_vtable channelz_node_arg_vtable = {
     channelz_node_copy, channelz_node_destroy, channelz_node_cmp};
-
-void CreateChannelzNode(ChannelStackBuilder* builder) {
-  auto args = builder->channel_args();
-  // Check whether channelz is enabled.
-  const bool channelz_enabled = args.GetBool(GRPC_ARG_ENABLE_CHANNELZ)
-                                    .value_or(GRPC_ENABLE_CHANNELZ_DEFAULT);
-  if (!channelz_enabled) return;
-  // Get parameters needed to create the channelz node.
-  const size_t channel_tracer_max_memory = std::max(
-      0, args.GetInt(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE)
-             .value_or(GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT));
-  const bool is_internal_channel =
-      args.GetBool(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL).value_or(false);
-  // Create the channelz node.
-  std::string target(builder->target());
-  RefCountedPtr<channelz::ChannelNode> channelz_node =
-      MakeRefCounted<channelz::ChannelNode>(
-          target.c_str(), channel_tracer_max_memory, is_internal_channel);
-  channelz_node->AddTraceEvent(
-      channelz::ChannelTrace::Severity::Info,
-      grpc_slice_from_static_string("Channel created"));
-  // Add channelz node to channel args.
-  // We remove the is_internal_channel arg, since we no longer need it.
-  builder->SetChannelArgs(
-      args.Remove(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL)
-          .Set(GRPC_ARG_CHANNELZ_CHANNEL_NODE,
-               ChannelArgs::Pointer(channelz_node.release(),
-                                    &channelz_node_arg_vtable)));
-}
-
 }  // namespace
 
 absl::StatusOr<RefCountedPtr<Channel>> Channel::Create(
     const char* target, ChannelArgs args,
     grpc_channel_stack_type channel_stack_type,
     grpc_transport* optional_transport) {
-  ChannelStackBuilderImpl builder(
-      grpc_channel_stack_type_string(channel_stack_type), channel_stack_type);
   if (!args.GetString(GRPC_ARG_DEFAULT_AUTHORITY).has_value()) {
     auto ssl_override = args.GetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG);
     if (ssl_override.has_value()) {
@@ -221,15 +190,42 @@ absl::StatusOr<RefCountedPtr<Channel>> Channel::Create(
       args = channel_args_mutator(target, args, channel_stack_type);
     }
   }
-  builder.SetChannelArgs(args).SetTarget(target).SetTransport(
-      optional_transport);
-  if (!CoreConfiguration::Get().channel_init().CreateStack(&builder)) {
-    return nullptr;
-  }
   // We only need to do this for clients here. For servers, this will be
   // done in src/core/lib/surface/server.cc.
   if (grpc_channel_stack_type_is_client(channel_stack_type)) {
-    CreateChannelzNode(&builder);
+    // Check whether channelz is enabled.
+    if (args.GetBool(GRPC_ARG_ENABLE_CHANNELZ)
+            .value_or(GRPC_ENABLE_CHANNELZ_DEFAULT)) {
+      // Get parameters needed to create the channelz node.
+      const size_t channel_tracer_max_memory = std::max(
+          0,
+          args.GetInt(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE)
+              .value_or(GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT));
+      const bool is_internal_channel =
+          args.GetBool(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL).value_or(false);
+      // Create the channelz node.
+      std::string channelz_node_target{target == nullptr ? "unknown" : target};
+      RefCountedPtr<channelz::ChannelNode> channelz_node =
+          MakeRefCounted<channelz::ChannelNode>(channelz_node_target,
+                                                channel_tracer_max_memory,
+                                                is_internal_channel);
+      channelz_node->AddTraceEvent(
+          channelz::ChannelTrace::Severity::Info,
+          grpc_slice_from_static_string("Channel created"));
+      // Add channelz node to channel args.
+      // We remove the is_internal_channel arg, since we no longer need it.
+      args = args.Remove(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL)
+                 .Set(GRPC_ARG_CHANNELZ_CHANNEL_NODE,
+                      ChannelArgs::Pointer(channelz_node.release(),
+                                           &channelz_node_arg_vtable));
+    }
+  }
+  ChannelStackBuilderImpl builder(
+      grpc_channel_stack_type_string(channel_stack_type), channel_stack_type,
+      args);
+  builder.SetTarget(target).SetTransport(optional_transport);
+  if (!CoreConfiguration::Get().channel_init().CreateStack(&builder)) {
+    return nullptr;
   }
   return CreateWithBuilder(&builder);
 }
@@ -320,9 +316,9 @@ grpc_call* grpc_channel_create_call(grpc_channel* channel,
   grpc_core::ExecCtx exec_ctx;
   grpc_call* call = grpc_channel_create_call_internal(
       channel, parent_call, propagation_mask, completion_queue, nullptr,
-      grpc_core::Slice(grpc_slice_ref_internal(method)),
+      grpc_core::Slice(grpc_core::CSliceRef(method)),
       host != nullptr
-          ? absl::optional<grpc_core::Slice>(grpc_slice_ref_internal(*host))
+          ? absl::optional<grpc_core::Slice>(grpc_core::CSliceRef(*host))
           : absl::nullopt,
       grpc_core::Timestamp::FromTimespecRoundUp(deadline));
 
@@ -338,7 +334,7 @@ grpc_call* grpc_channel_create_pollset_set_call(
       channel, parent_call, propagation_mask, nullptr, pollset_set,
       grpc_core::Slice(method),
       host != nullptr
-          ? absl::optional<grpc_core::Slice>(grpc_slice_ref_internal(*host))
+          ? absl::optional<grpc_core::Slice>(grpc_core::CSliceRef(*host))
           : absl::nullopt,
       deadline);
 }
@@ -429,8 +425,7 @@ void grpc_channel_destroy_internal(grpc_channel* c_channel) {
   grpc_transport_op* op = grpc_make_transport_op(nullptr);
   grpc_channel_element* elem;
   GRPC_API_TRACE("grpc_channel_destroy(channel=%p)", 1, (c_channel));
-  op->disconnect_with_error =
-      GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel Destroyed");
+  op->disconnect_with_error = GRPC_ERROR_CREATE("Channel Destroyed");
   elem = grpc_channel_stack_element(channel->channel_stack(), 0);
   elem->filter->start_transport_op(elem, op);
 }

data/src/core/lib/surface/channel.h

@@ -34,6 +34,7 @@
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 
+#include <grpc/event_engine/event_engine.h>
 #include <grpc/event_engine/memory_allocator.h>
 #include <grpc/impl/codegen/compression_types.h>
 #include <grpc/impl/codegen/grpc_types.h>
@@ -141,6 +142,7 @@ class Channel : public RefCounted<Channel>,
   absl::string_view target() const { return target_; }
   MemoryAllocator* allocator() { return &allocator_; }
   bool is_client() const { return is_client_; }
+  bool is_promising() const { return is_promising_; }
   RegisteredCall* RegisterCall(const char* method, const char* host);
 
   int TestOnlyRegisteredCalls() {
@@ -153,12 +155,18 @@ class Channel : public RefCounted<Channel>,
     return registration_table_.method_registration_attempts;
   }
 
+  grpc_event_engine::experimental::EventEngine* event_engine() const {
+    return channel_stack_->EventEngine();
+  }
+
  private:
-  Channel(bool is_client, std::string target, const ChannelArgs& channel_args,
+  Channel(bool is_client, bool is_promising, std::string target,
+          const ChannelArgs& channel_args,
           grpc_compression_options compression_options,
           RefCountedPtr<grpc_channel_stack> channel_stack);
 
   const bool is_client_;
+  const bool is_promising_;
   const grpc_compression_options compression_options_;
   std::atomic<size_t> call_size_estimate_;
   CallRegistrationTable registration_table_;

data/src/core/lib/surface/channel_ping.cc

@@ -45,7 +45,7 @@ static void ping_destroy(void* arg, grpc_cq_completion* /*storage*/) {
 
 static void ping_done(void* arg, grpc_error_handle error) {
   ping_result* pr = static_cast<ping_result*>(arg);
-  grpc_cq_end_op(pr->cq, pr->tag, GRPC_ERROR_REF(error), ping_destroy, pr,
+  grpc_cq_end_op(pr->cq, pr->tag, error, ping_destroy, pr,
                  &pr->completion_storage);
 }
 

data/src/core/lib/surface/channel_stack_type.cc

@@ -30,6 +30,8 @@ bool grpc_channel_stack_type_is_client(grpc_channel_stack_type type) {
       return true;
     case GRPC_CLIENT_DIRECT_CHANNEL:
       return true;
+    case GRPC_CLIENT_DYNAMIC:
+      return true;
     case GRPC_SERVER_CHANNEL:
       return false;
     case GRPC_NUM_CHANNEL_STACK_TYPES:
@@ -50,6 +52,8 @@ const char* grpc_channel_stack_type_string(grpc_channel_stack_type type) {
       return "CLIENT_LAME_CHANNEL";
     case GRPC_CLIENT_DIRECT_CHANNEL:
       return "CLIENT_DIRECT_CHANNEL";
+    case GRPC_CLIENT_DYNAMIC:
+      return "CLIENT_DYNAMIC_CHANNEL";
     case GRPC_NUM_CHANNEL_STACK_TYPES:
       break;
   }

data/src/core/lib/surface/channel_stack_type.h

@@ -27,6 +27,8 @@ typedef enum {
   // bottom-half of a client channel: everything that happens post-load
   // balancing (bound to a specific transport)
   GRPC_CLIENT_SUBCHANNEL,
+  // dynamic part of a client channel
+  GRPC_CLIENT_DYNAMIC,
   // a permanently broken client channel
   GRPC_CLIENT_LAME_CHANNEL,
   // a directly connected client channel (without load-balancing, directly talks