grpc 1.63.0 → 1.65.2

data/src/core/client_channel/load_balanced_call_destination.cc

@@ -0,0 +1,336 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/core/client_channel/load_balanced_call_destination.h"
+
+#include "src/core/client_channel/client_channel.h"
+#include "src/core/client_channel/client_channel_internal.h"
+#include "src/core/client_channel/subchannel.h"
+#include "src/core/lib/channel/status_util.h"
+#include "src/core/lib/config/core_configuration.h"
+#include "src/core/lib/promise/loop.h"
+#include "src/core/telemetry/call_tracer.h"
+
+namespace grpc_core {
+
+namespace {
+
+class LbMetadata : public LoadBalancingPolicy::MetadataInterface {
+ public:
+  explicit LbMetadata(grpc_metadata_batch* batch) : batch_(batch) {}
+
+  void Add(absl::string_view key, absl::string_view value) override {
+    if (batch_ == nullptr) return;
+    // Gross, egregious hack to support legacy grpclb behavior.
+    // TODO(ctiller): Use a promise context for this once that plumbing is done.
+    if (key == GrpcLbClientStatsMetadata::key()) {
+      batch_->Set(
+          GrpcLbClientStatsMetadata(),
+          const_cast<GrpcLbClientStats*>(
+              reinterpret_cast<const GrpcLbClientStats*>(value.data())));
+      return;
+    }
+    batch_->Append(key, Slice::FromStaticString(value),
+                   [key](absl::string_view error, const Slice& value) {
+                     gpr_log(GPR_ERROR, "%s",
+                             absl::StrCat(error, " key:", key,
+                                          " value:", value.as_string_view())
+                                 .c_str());
+                   });
+  }
+
+  std::vector<std::pair<std::string, std::string>> TestOnlyCopyToVector()
+      override {
+    if (batch_ == nullptr) return {};
+    Encoder encoder;
+    batch_->Encode(&encoder);
+    return encoder.Take();
+  }
+
+  absl::optional<absl::string_view> Lookup(absl::string_view key,
+                                           std::string* buffer) const override {
+    if (batch_ == nullptr) return absl::nullopt;
+    return batch_->GetStringValue(key, buffer);
+  }
+
+ private:
+  class Encoder {
+   public:
+    void Encode(const Slice& key, const Slice& value) {
+      out_.emplace_back(std::string(key.as_string_view()),
+                        std::string(value.as_string_view()));
+    }
+
+    template <class Which>
+    void Encode(Which, const typename Which::ValueType& value) {
+      auto value_slice = Which::Encode(value);
+      out_.emplace_back(std::string(Which::key()),
+                        std::string(value_slice.as_string_view()));
+    }
+
+    void Encode(GrpcTimeoutMetadata,
+                const typename GrpcTimeoutMetadata::ValueType&) {}
+    void Encode(HttpPathMetadata, const Slice&) {}
+    void Encode(HttpMethodMetadata,
+                const typename HttpMethodMetadata::ValueType&) {}
+
+    std::vector<std::pair<std::string, std::string>> Take() {
+      return std::move(out_);
+    }
+
+   private:
+    std::vector<std::pair<std::string, std::string>> out_;
+  };
+
+  grpc_metadata_batch* batch_;
+};
+
+void MaybeCreateCallAttemptTracer(bool is_transparent_retry) {
+  auto* call_tracer = MaybeGetContext<ClientCallTracer>();
+  if (call_tracer == nullptr) return;
+  auto* tracer = call_tracer->StartNewAttempt(is_transparent_retry);
+  SetContext<CallTracerInterface>(tracer);
+}
+
+class LbCallState : public ClientChannelLbCallState {
+ public:
+  void* Alloc(size_t size) override { return GetContext<Arena>()->Alloc(size); }
+
+  // Internal API to allow first-party LB policies to access per-call
+  // attributes set by the ConfigSelector.
+  ServiceConfigCallData::CallAttributeInterface* GetCallAttribute(
+      UniqueTypeName type) const override {
+    auto* service_config_call_data = GetContext<ServiceConfigCallData>();
+    return service_config_call_data->GetCallAttribute(type);
+  }
+
+  ClientCallTracer::CallAttemptTracer* GetCallAttemptTracer() const override {
+    return GetContext<ClientCallTracer::CallAttemptTracer>();
+  }
+};
+
+// TODO(roth): Remove this in favor of the gprpp Match() function once
+// we can do that without breaking lock annotations.
+template <typename T>
+T HandlePickResult(
+    LoadBalancingPolicy::PickResult* result,
+    std::function<T(LoadBalancingPolicy::PickResult::Complete*)> complete_func,
+    std::function<T(LoadBalancingPolicy::PickResult::Queue*)> queue_func,
+    std::function<T(LoadBalancingPolicy::PickResult::Fail*)> fail_func,
+    std::function<T(LoadBalancingPolicy::PickResult::Drop*)> drop_func) {
+  auto* complete_pick =
+      absl::get_if<LoadBalancingPolicy::PickResult::Complete>(&result->result);
+  if (complete_pick != nullptr) {
+    return complete_func(complete_pick);
+  }
+  auto* queue_pick =
+      absl::get_if<LoadBalancingPolicy::PickResult::Queue>(&result->result);
+  if (queue_pick != nullptr) {
+    return queue_func(queue_pick);
+  }
+  auto* fail_pick =
+      absl::get_if<LoadBalancingPolicy::PickResult::Fail>(&result->result);
+  if (fail_pick != nullptr) {
+    return fail_func(fail_pick);
+  }
+  auto* drop_pick =
+      absl::get_if<LoadBalancingPolicy::PickResult::Drop>(&result->result);
+  CHECK(drop_pick != nullptr);
+  return drop_func(drop_pick);
+}
+
+// Does an LB pick for a call.  Returns one of the following things:
+// - Continue{}, meaning to queue the pick
+// - a non-OK status, meaning to fail the call
+// - a call destination, meaning that the pick is complete
+// When the pick is complete, pushes client_initial_metadata onto
+// call_initiator.  Also adds the subchannel call tracker (if any) to
+// context.
+LoopCtl<absl::StatusOr<RefCountedPtr<UnstartedCallDestination>>> PickSubchannel(
+    LoadBalancingPolicy::SubchannelPicker& picker,
+    UnstartedCallHandler& unstarted_handler) {
+  // Perform LB pick.
+  auto& client_initial_metadata =
+      unstarted_handler.UnprocessedClientInitialMetadata();
+  LoadBalancingPolicy::PickArgs pick_args;
+  Slice* path = client_initial_metadata.get_pointer(HttpPathMetadata());
+  CHECK(path != nullptr);
+  pick_args.path = path->as_string_view();
+  LbCallState lb_call_state;
+  pick_args.call_state = &lb_call_state;
+  LbMetadata initial_metadata(&client_initial_metadata);
+  pick_args.initial_metadata = &initial_metadata;
+  auto result = picker.Pick(pick_args);
+  // Handle result.
+  return HandlePickResult<
+      LoopCtl<absl::StatusOr<RefCountedPtr<UnstartedCallDestination>>>>(
+      &result,
+      // CompletePick
+      [&](LoadBalancingPolicy::PickResult::Complete* complete_pick)
+          -> LoopCtl<absl::StatusOr<RefCountedPtr<UnstartedCallDestination>>> {
+        GRPC_TRACE_LOG(client_channel_lb_call, INFO)
+            << "client_channel: " << GetContext<Activity>()->DebugTag()
+            << " pick succeeded: subchannel="
+            << complete_pick->subchannel.get();
+        CHECK(complete_pick->subchannel != nullptr);
+        // Grab a ref to the call destination while we're still
+        // holding the data plane mutex.
+        auto call_destination =
+            DownCast<SubchannelInterfaceWithCallDestination*>(
+                complete_pick->subchannel.get())
+                ->call_destination();
+        // If the subchannel has no call destination (e.g., if the
+        // subchannel has moved out of state READY but the LB policy hasn't
+        // yet seen that change and given us a new picker), then just
+        // queue the pick.  We'll try again as soon as we get a new picker.
+        if (call_destination == nullptr) {
+          GRPC_TRACE_LOG(client_channel_lb_call, INFO)
+              << "client_channel: " << GetContext<Activity>()->DebugTag()
+              << " returned by LB picker has no connected subchannel; queueing "
+                 "pick";
+          return Continue{};
+        }
+        // If the LB policy returned a call tracker, inform it that the
+        // call is starting and add it to context, so that we can notify
+        // it when the call finishes.
+        if (complete_pick->subchannel_call_tracker != nullptr) {
+          complete_pick->subchannel_call_tracker->Start();
+          SetContext(complete_pick->subchannel_call_tracker.release());
+        }
+        // Return the connected subchannel.
+        return call_destination;
+      },
+      // QueuePick
+      [&](LoadBalancingPolicy::PickResult::Queue* /*queue_pick*/) {
+        GRPC_TRACE_LOG(client_channel_lb_call, INFO)
+            << "client_channel: " << GetContext<Activity>()->DebugTag()
+            << " pick queued";
+        return Continue{};
+      },
+      // FailPick
+      [&](LoadBalancingPolicy::PickResult::Fail* fail_pick)
+          -> LoopCtl<absl::StatusOr<RefCountedPtr<UnstartedCallDestination>>> {
+        GRPC_TRACE_LOG(client_channel_lb_call, INFO)
+            << "client_channel: " << GetContext<Activity>()->DebugTag()
+            << " pick failed: " << fail_pick->status;
+        // If wait_for_ready is false, then the error indicates the RPC
+        // attempt's final status.
+        if (!unstarted_handler.UnprocessedClientInitialMetadata()
+                 .GetOrCreatePointer(WaitForReady())
+                 ->value) {
+          return MaybeRewriteIllegalStatusCode(std::move(fail_pick->status),
+                                               "LB pick");
+        }
+        // If wait_for_ready is true, then queue to retry when we get a new
+        // picker.
+        return Continue{};
+      },
+      // DropPick
+      [&](LoadBalancingPolicy::PickResult::Drop* drop_pick)
+          -> LoopCtl<absl::StatusOr<RefCountedPtr<UnstartedCallDestination>>> {
+        GRPC_TRACE_LOG(client_channel_lb_call, INFO)
+            << "client_channel: " << GetContext<Activity>()->DebugTag()
+            << " pick dropped: " << drop_pick->status;
+        return grpc_error_set_int(MaybeRewriteIllegalStatusCode(
+                                      std::move(drop_pick->status), "LB drop"),
+                                  StatusIntProperty::kLbPolicyDrop, 1);
+      });
+}
+
+}  // namespace
+
+void LoadBalancedCallDestination::StartCall(
+    UnstartedCallHandler unstarted_handler) {
+  // If there is a call tracer, create a call attempt tracer.
+  bool* is_transparent_retry_metadata =
+      unstarted_handler.UnprocessedClientInitialMetadata().get_pointer(
+          IsTransparentRetry());
+  bool is_transparent_retry = is_transparent_retry_metadata != nullptr
+                                  ? *is_transparent_retry_metadata
+                                  : false;
+  MaybeCreateCallAttemptTracer(is_transparent_retry);
+  // Spawn a promise to do the LB pick.
+  // This will eventually start the call.
+  unstarted_handler.SpawnGuardedUntilCallCompletes(
+      "lb_pick", [unstarted_handler, picker = picker_]() mutable {
+        return Map(
+            // Wait for the LB picker.
+            CheckDelayed(Loop(
+                [last_picker =
+                     RefCountedPtr<LoadBalancingPolicy::SubchannelPicker>(),
+                 unstarted_handler, picker]() mutable {
+                  return Map(
+                      picker.Next(last_picker),
+                      [unstarted_handler, &last_picker](
+                          RefCountedPtr<LoadBalancingPolicy::SubchannelPicker>
+                              picker) mutable {
+                        CHECK_NE(picker.get(), nullptr);
+                        last_picker = std::move(picker);
+                        // Returns 3 possible things:
+                        // - Continue to queue the pick
+                        // - non-OK status to fail the pick
+                        // - a connected subchannel to complete the pick
+                        return PickSubchannel(*last_picker, unstarted_handler);
+                      });
+                })),
+            // Create call stack on the connected subchannel.
+            [unstarted_handler](
+                std::tuple<
+                    absl::StatusOr<RefCountedPtr<UnstartedCallDestination>>,
+                    bool>
+                    pick_result) {
+              auto& call_destination = std::get<0>(pick_result);
+              const bool was_queued = std::get<1>(pick_result);
+              if (!call_destination.ok()) {
+                return call_destination.status();
+              }
+              // LB pick is done, so indicate that we've committed.
+              auto* on_commit = MaybeGetContext<LbOnCommit>();
+              if (on_commit != nullptr && *on_commit != nullptr) {
+                (*on_commit)();
+              }
+              // If it was queued, add a trace annotation.
+              if (was_queued) {
+                auto* tracer =
+                    MaybeGetContext<ClientCallTracer::CallAttemptTracer>();
+                if (tracer != nullptr) {
+                  tracer->RecordAnnotation("Delayed LB pick complete.");
+                }
+              }
+              // Delegate to connected subchannel.
+              // TODO(ctiller): need to insert LbCallTracingFilter at the top of
+              // the stack
+              (*call_destination)->StartCall(unstarted_handler);
+              return absl::OkStatus();
+            });
+      });
+}
+
+void RegisterLoadBalancedCallDestination(CoreConfiguration::Builder* builder) {
+  class LoadBalancedCallDestinationFactory final
+      : public ClientChannel::CallDestinationFactory {
+   public:
+    RefCountedPtr<UnstartedCallDestination> CreateCallDestination(
+        ClientChannel::PickerObservable picker) override {
+      return MakeRefCounted<LoadBalancedCallDestination>(std::move(picker));
+    }
+  };
+
+  builder->channel_args_preconditioning()->RegisterStage([](ChannelArgs args) {
+    return args.SetObject(
+        NoDestructSingleton<LoadBalancedCallDestinationFactory>::Get());
+  });
+}
+
+}  // namespace grpc_core

data/src/core/client_channel/load_balanced_call_destination.h

@@ -0,0 +1,49 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_CLIENT_CHANNEL_LOAD_BALANCED_CALL_DESTINATION_H
+#define GRPC_SRC_CORE_CLIENT_CHANNEL_LOAD_BALANCED_CALL_DESTINATION_H
+
+#include "absl/functional/any_invocable.h"
+
+#include "src/core/client_channel/client_channel.h"
+#include "src/core/lib/promise/context.h"
+#include "src/core/lib/transport/call_destination.h"
+#include "src/core/load_balancing/lb_policy.h"
+
+namespace grpc_core {
+
+// Context type for LB on_commit callback.
+// TODO(ctiller): make this a struct, so we don't accidentally alias context
+// types
+using LbOnCommit = absl::AnyInvocable<void()>;
+template <>
+struct ContextType<LbOnCommit> {};
+
+class LoadBalancedCallDestination final : public UnstartedCallDestination {
+ public:
+  explicit LoadBalancedCallDestination(ClientChannel::PickerObservable picker)
+      : picker_(std::move(picker)) {}
+
+  void Orphaned() override {}
+
+  void StartCall(UnstartedCallHandler unstarted_handler) override;
+
+ private:
+  ClientChannel::PickerObservable picker_;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_CLIENT_CHANNEL_LOAD_BALANCED_CALL_DESTINATION_H

data/src/core/client_channel/local_subchannel_pool.cc

@@ -22,6 +22,8 @@
 
 #include <utility>
 
+#include "absl/log/check.h"
+
 #include <grpc/support/log.h>
 
 #include "src/core/client_channel/subchannel.h"
@@ -34,7 +36,7 @@ RefCountedPtr<Subchannel> LocalSubchannelPool::RegisterSubchannel(
   // Because this pool is only accessed under the client channel's work
   // serializer, and because FindSubchannel is checked before invoking
   // RegisterSubchannel, no such subchannel should exist in the map.
-  GPR_ASSERT(it == subchannel_map_.end());
+  CHECK(it == subchannel_map_.end());
   subchannel_map_[key] = constructed.get();
   return constructed;
 }
@@ -45,8 +47,8 @@ void LocalSubchannelPool::UnregisterSubchannel(const SubchannelKey& key,
   // Because this subchannel pool is accessed only under the client
   // channel's work serializer, any subchannel created by RegisterSubchannel
   // will be deleted from the map in UnregisterSubchannel.
-  GPR_ASSERT(it != subchannel_map_.end());
-  GPR_ASSERT(it->second == subchannel);
+  CHECK(it != subchannel_map_.end());
+  CHECK(it->second == subchannel);
   subchannel_map_.erase(it);
 }
 

data/src/core/client_channel/retry_filter.cc

@@ -35,9 +35,9 @@
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/uri/uri_parser.h"
 #include "src/core/service_config/service_config.h"
 #include "src/core/service_config/service_config_call_data.h"
-#include "src/core/lib/uri/uri_parser.h"
 
 //
 // Retry filter
@@ -89,8 +89,6 @@ using grpc_core::internal::RetryMethodConfig;
 using grpc_core::internal::RetryServiceConfigParser;
 using grpc_event_engine::experimental::EventEngine;
 
-grpc_core::TraceFlag grpc_retry_trace(false, "retry");
-
 namespace grpc_core {
 
 //
@@ -130,11 +128,8 @@ RetryFilter::RetryFilter(const ChannelArgs& args, grpc_error_handle* error)
           server_name, config->max_milli_tokens(), config->milli_token_ratio());
 }
 
-const RetryMethodConfig* RetryFilter::GetRetryPolicy(
-    const grpc_call_context_element* context) {
-  if (context == nullptr) return nullptr;
-  auto* svc_cfg_call_data = static_cast<ServiceConfigCallData*>(
-      context[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value);
+const RetryMethodConfig* RetryFilter::GetRetryPolicy(Arena* arena) {
+  auto* svc_cfg_call_data = arena->GetContext<ServiceConfigCallData>();
   if (svc_cfg_call_data == nullptr) return nullptr;
   return static_cast<const RetryMethodConfig*>(
       svc_cfg_call_data->GetMethodParsedConfig(service_config_parser_index_));
@@ -142,8 +137,6 @@ const RetryMethodConfig* RetryFilter::GetRetryPolicy(
 
 const grpc_channel_filter RetryFilter::kVtable = {
     RetryFilter::LegacyCallData::StartTransportStreamOpBatch,
-    nullptr,
-    /* init_call: */ nullptr,
     RetryFilter::StartTransportOp,
     sizeof(RetryFilter::LegacyCallData),
     RetryFilter::LegacyCallData::Init,

data/src/core/client_channel/retry_filter.h

@@ -24,6 +24,7 @@
 
 #include <new>
 
+#include "absl/log/check.h"
 #include "absl/types/optional.h"
 
 #include <grpc/event_engine/event_engine.h>
@@ -37,14 +38,10 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/channel/context.h"
-#include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/transport/transport.h"
-
-extern grpc_core::TraceFlag grpc_retry_trace;
+#include "src/core/util/useful.h"
 
 namespace grpc_core {
 
@@ -65,8 +62,7 @@ class RetryFilter final {
   // any even moderately compelling reason to do so.
   static double BackoffJitter() { return 0.2; }
 
-  const internal::RetryMethodConfig* GetRetryPolicy(
-      const grpc_call_context_element* context);
+  const internal::RetryMethodConfig* GetRetryPolicy(Arena* arena);
 
   RefCountedPtr<internal::ServerRetryThrottleData> retry_throttle_data() const {
     return retry_throttle_data_;
@@ -92,8 +88,8 @@ class RetryFilter final {
 
   static grpc_error_handle Init(grpc_channel_element* elem,
                                 grpc_channel_element_args* args) {
-    GPR_ASSERT(args->is_last);
-    GPR_ASSERT(elem->filter == &kVtable);
+    CHECK(args->is_last);
+    CHECK(elem->filter == &kVtable);
     grpc_error_handle error;
     new (elem->channel_data) RetryFilter(args->channel_args, &error);
     return error;