grpc 1.58.0.pre1 → 1.59.0

data/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc

@@ -123,7 +123,8 @@ class CdsLb : public LoadBalancingPolicy {
     ClusterWatcher(RefCountedPtr<CdsLb> parent, std::string name)
         : parent_(std::move(parent)), name_(std::move(name)) {}
 
-    void OnResourceChanged(XdsClusterResource cluster_data) override {
+    void OnResourceChanged(
+        std::shared_ptr<const XdsClusterResource> cluster_data) override {
       RefCountedPtr<ClusterWatcher> self = Ref();
       parent_->work_serializer()->Run(
           [self = std::move(self),
@@ -160,7 +161,7 @@ class CdsLb : public LoadBalancingPolicy {
     // Not owned, so do not dereference.
     ClusterWatcher* watcher = nullptr;
     // Most recent update obtained from this watcher.
-    absl::optional<XdsClusterResource> update;
+    std::shared_ptr<const XdsClusterResource> update;
   };
 
   // Delegating helper to be passed to child policy.
@@ -174,7 +175,7 @@ class CdsLb : public LoadBalancingPolicy {
       const std::string& name, int depth, Json::Array* discovery_mechanisms,
       std::set<std::string>* clusters_added);
   void OnClusterChanged(const std::string& name,
-                        XdsClusterResource cluster_data);
+                        std::shared_ptr<const XdsClusterResource> cluster_data);
   void OnError(const std::string& name, absl::Status status);
   void OnResourceDoesNotExist(const std::string& name);
 
@@ -328,7 +329,7 @@ absl::StatusOr<bool> CdsLb::GenerateDiscoveryMechanismForCluster(
     return false;
   }
   // Don't have the update we need yet.
-  if (!state.update.has_value()) return false;
+  if (state.update == nullptr) return false;
   // For AGGREGATE clusters, recursively expand to child clusters.
   auto* aggregate =
       absl::get_if<XdsClusterResource::Aggregate>(&state.update->type);
@@ -422,13 +423,15 @@ absl::StatusOr<bool> CdsLb::GenerateDiscoveryMechanismForCluster(
   return true;
 }
 
-void CdsLb::OnClusterChanged(const std::string& name,
-                             XdsClusterResource cluster_data) {
+void CdsLb::OnClusterChanged(
+    const std::string& name,
+    std::shared_ptr<const XdsClusterResource> cluster_data) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) {
     gpr_log(
         GPR_INFO,
         "[cdslb %p] received CDS update for cluster %s from xds client %p: %s",
-        this, name.c_str(), xds_client_.get(), cluster_data.ToString().c_str());
+        this, name.c_str(), xds_client_.get(),
+        cluster_data->ToString().c_str());
   }
   // Store the update in the map if we are still interested in watching this
   // cluster (i.e., it is not cancelled already).
@@ -436,10 +439,9 @@ void CdsLb::OnClusterChanged(const std::string& name,
   // that was scheduled before the deletion, so we can just ignore it.
   auto it = watchers_.find(name);
   if (it == watchers_.end()) return;
-  it->second.update = cluster_data;
+  it->second.update = std::move(cluster_data);
   // Take care of integration with new certificate code.
-  absl::Status status =
-      UpdateXdsCertificateProvider(name, it->second.update.value());
+  absl::Status status = UpdateXdsCertificateProvider(name, *it->second.update);
   if (!status.ok()) {
     return OnError(name, status);
   }
@@ -455,6 +457,10 @@ void CdsLb::OnClusterChanged(const std::string& name,
     return OnError(name, result.status());
   }
   if (*result) {
+    if (discovery_mechanisms.empty()) {
+      return OnError(name, absl::FailedPreconditionError(
+                               "aggregate cluster graph has no leaf clusters"));
+    }
     // LB policy is configured by aggregate cluster, not by the individual
     // underlying cluster that we may be processing an update for.
     auto it = watchers_.find(config_->cluster());

data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc

@@ -54,6 +54,7 @@
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/gprpp/no_destruct.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/ref_counted_string.h"
@@ -212,7 +213,8 @@ class XdsClusterResolverLb : public LoadBalancingPolicy {
       ~EndpointWatcher() override {
         discovery_mechanism_.reset(DEBUG_LOCATION, "EndpointWatcher");
       }
-      void OnResourceChanged(XdsEndpointResource update) override {
+      void OnResourceChanged(
+          std::shared_ptr<const XdsEndpointResource> update) override {
         RefCountedPtr<EndpointWatcher> self = Ref();
         discovery_mechanism_->parent()->work_serializer()->Run(
             [self = std::move(self), update = std::move(update)]() mutable {
@@ -241,15 +243,16 @@ class XdsClusterResolverLb : public LoadBalancingPolicy {
       // Code accessing protected methods of `DiscoveryMechanism` need to be
       // in methods of this class rather than in lambdas to work around an MSVC
       // bug.
-      void OnResourceChangedHelper(XdsEndpointResource update) {
+      void OnResourceChangedHelper(
+          std::shared_ptr<const XdsEndpointResource> update) {
         std::string resolution_note;
-        if (update.priorities.empty()) {
+        if (update->priorities.empty()) {
           resolution_note = absl::StrCat(
               "EDS resource ", discovery_mechanism_->GetEdsResourceName(),
               " contains no localities");
         } else {
           std::set<std::string> empty_localities;
-          for (const auto& priority : update.priorities) {
+          for (const auto& priority : update->priorities) {
             for (const auto& p : priority.localities) {
               if (p.second.endpoints.empty()) {
                 empty_localities.insert(p.first->AsHumanReadableString());
@@ -345,7 +348,7 @@ class XdsClusterResolverLb : public LoadBalancingPolicy {
   struct DiscoveryMechanismEntry {
     OrphanablePtr<DiscoveryMechanism> discovery_mechanism;
     // Most recent update reported by the discovery mechanism.
-    absl::optional<XdsEndpointResource> latest_update;
+    std::shared_ptr<const XdsEndpointResource> latest_update;
     // Last resolution note reported by the discovery mechanism, if any.
     std::string resolution_note;
     // State used to retain child policy names for priority policy.
@@ -376,7 +379,8 @@ class XdsClusterResolverLb : public LoadBalancingPolicy {
 
   void ShutdownLocked() override;
 
-  void OnEndpointChanged(size_t index, XdsEndpointResource update,
+  void OnEndpointChanged(size_t index,
+                         std::shared_ptr<const XdsEndpointResource> update,
                          std::string resolution_note);
   void OnError(size_t index, std::string resolution_note);
   void OnResourceDoesNotExist(size_t index, std::string resolution_note);
@@ -503,14 +507,14 @@ void XdsClusterResolverLb::LogicalDNSDiscoveryMechanism::ResolverResultHandler::
     return;
   }
   // Convert resolver result to EDS update.
-  XdsEndpointResource update;
+  auto update = std::make_shared<XdsEndpointResource>();
   XdsEndpointResource::Priority::Locality locality;
   locality.name = MakeRefCounted<XdsLocalityName>("", "", "");
   locality.lb_weight = 1;
   locality.endpoints = std::move(*result.addresses);
   XdsEndpointResource::Priority priority;
   priority.localities.emplace(locality.name.get(), std::move(locality));
-  update.priorities.emplace_back(std::move(priority));
+  update->priorities.emplace_back(std::move(priority));
   lb_policy->OnEndpointChanged(index, std::move(update),
                                std::move(result.resolution_note));
 }
@@ -623,9 +627,21 @@ void XdsClusterResolverLb::ExitIdleLocked() {
   if (child_policy_ != nullptr) child_policy_->ExitIdleLocked();
 }
 
-void XdsClusterResolverLb::OnEndpointChanged(size_t index,
-                                             XdsEndpointResource update,
-                                             std::string resolution_note) {
+// We need at least one priority for each discovery mechanism, just so that we
+// have a child in which to create the xds_cluster_impl policy.  This ensures
+// that we properly handle the case of a discovery mechanism dropping 100% of
+// calls, the OnError() case, and the OnResourceDoesNotExist() case.
+const XdsEndpointResource::PriorityList& GetUpdatePriorityList(
+    const XdsEndpointResource& update) {
+  static const NoDestruct<XdsEndpointResource::PriorityList>
+      kPriorityListWithEmptyPriority(1);
+  if (update.priorities.empty()) return *kPriorityListWithEmptyPriority;
+  return update.priorities;
+}
+
+void XdsClusterResolverLb::OnEndpointChanged(
+    size_t index, std::shared_ptr<const XdsEndpointResource> update,
+    std::string resolution_note) {
   if (shutting_down_) return;
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) {
     gpr_log(GPR_INFO,
@@ -634,11 +650,8 @@ void XdsClusterResolverLb::OnEndpointChanged(size_t index,
             this, index, resolution_note.c_str());
   }
   DiscoveryMechanismEntry& discovery_entry = discovery_mechanisms_[index];
-  // We need at least one priority for each discovery mechanism, just so that we
-  // have a child in which to create the xds_cluster_impl policy.  This ensures
-  // that we properly handle the case of a discovery mechanism dropping 100% of
-  // calls, the OnError() case, and the OnResourceDoesNotExist() case.
-  if (update.priorities.empty()) update.priorities.emplace_back();
+  const XdsEndpointResource::PriorityList& priority_list =
+      GetUpdatePriorityList(*update);
   // Update priority_child_numbers, reusing old child numbers in an
   // intelligent way to avoid unnecessary churn.
   // First, build some maps from locality to child number and the reverse
@@ -647,8 +660,9 @@ void XdsClusterResolverLb::OnEndpointChanged(size_t index,
       locality_child_map;
   std::map<size_t, std::set<XdsLocalityName*, XdsLocalityName::Less>>
       child_locality_map;
-  if (discovery_entry.latest_update.has_value()) {
-    const auto& prev_priority_list = discovery_entry.latest_update->priorities;
+  if (discovery_entry.latest_update != nullptr) {
+    const auto& prev_priority_list =
+        GetUpdatePriorityList(*discovery_entry.latest_update);
     for (size_t priority = 0; priority < prev_priority_list.size();
          ++priority) {
       size_t child_number = discovery_entry.priority_child_numbers[priority];
@@ -662,8 +676,8 @@ void XdsClusterResolverLb::OnEndpointChanged(size_t index,
   }
   // Construct new list of children.
   std::vector<size_t> priority_child_numbers;
-  for (size_t priority = 0; priority < update.priorities.size(); ++priority) {
-    const auto& localities = update.priorities[priority].localities;
+  for (size_t priority = 0; priority < priority_list.size(); ++priority) {
+    const auto& localities = priority_list[priority].localities;
     absl::optional<size_t> child_number;
     // If one of the localities in this priority already existed, reuse its
     // child number.
@@ -716,7 +730,7 @@ void XdsClusterResolverLb::OnEndpointChanged(size_t index,
   // will put the channel into TRANSIENT_FAILURE instead of CONNECTING
   // while we're still waiting for the other discovery mechanism(s).
   for (DiscoveryMechanismEntry& mechanism : discovery_mechanisms_) {
-    if (!mechanism.latest_update.has_value()) return;
+    if (mechanism.latest_update == nullptr) return;
   }
   // Update child policy.
   // TODO(roth): If the child policy reports an error with the update,
@@ -730,10 +744,11 @@ void XdsClusterResolverLb::OnError(size_t index, std::string resolution_note) {
           " reported error: %s",
           this, index, resolution_note.c_str());
   if (shutting_down_) return;
-  if (!discovery_mechanisms_[index].latest_update.has_value()) {
+  if (discovery_mechanisms_[index].latest_update == nullptr) {
     // Call OnEndpointChanged() with an empty update just like
     // OnResourceDoesNotExist().
-    OnEndpointChanged(index, XdsEndpointResource(), std::move(resolution_note));
+    OnEndpointChanged(index, std::make_shared<XdsEndpointResource>(),
+                      std::move(resolution_note));
   }
 }
 
@@ -745,7 +760,8 @@ void XdsClusterResolverLb::OnResourceDoesNotExist(size_t index,
           this, index, resolution_note.c_str());
   if (shutting_down_) return;
   // Call OnEndpointChanged() with an empty update.
-  OnEndpointChanged(index, XdsEndpointResource(), std::move(resolution_note));
+  OnEndpointChanged(index, std::make_shared<XdsEndpointResource>(),
+                    std::move(resolution_note));
 }
 
 //
@@ -755,11 +771,10 @@ void XdsClusterResolverLb::OnResourceDoesNotExist(size_t index,
 ServerAddressList XdsClusterResolverLb::CreateChildPolicyAddressesLocked() {
   ServerAddressList addresses;
   for (const auto& discovery_entry : discovery_mechanisms_) {
-    for (size_t priority = 0;
-         priority < discovery_entry.latest_update->priorities.size();
-         ++priority) {
-      const auto& priority_entry =
-          discovery_entry.latest_update->priorities[priority];
+    const auto& priority_list =
+        GetUpdatePriorityList(*discovery_entry.latest_update);
+    for (size_t priority = 0; priority < priority_list.size(); ++priority) {
+      const auto& priority_entry = priority_list[priority];
       std::string priority_child_name =
           discovery_entry.GetChildPolicyName(priority);
       for (const auto& p : priority_entry.localities) {
@@ -803,10 +818,10 @@ XdsClusterResolverLb::CreateChildPolicyConfigLocked() {
   Json::Object priority_children;
   Json::Array priority_priorities;
   for (const auto& discovery_entry : discovery_mechanisms_) {
+    const auto& priority_list =
+        GetUpdatePriorityList(*discovery_entry.latest_update);
     const auto& discovery_config = discovery_entry.config();
-    for (size_t priority = 0;
-         priority < discovery_entry.latest_update->priorities.size();
-         ++priority) {
+    for (size_t priority = 0; priority < priority_list.size(); ++priority) {
       // Determine what xDS LB policy to use.
       Json child_policy;
       if (!discovery_entry.discovery_mechanism->override_child_policy()

data/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc

@@ -18,6 +18,7 @@
 
 #include "src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h"
 
+#include <inttypes.h>
 #include <stddef.h>
 
 #include <algorithm>
@@ -28,6 +29,7 @@
 #include <set>
 #include <string>
 #include <tuple>
+#include <type_traits>
 #include <utility>
 #include <vector>
 
@@ -50,6 +52,7 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/gprpp/match.h"
 #include "src/core/lib/gprpp/orphanable.h"
@@ -430,7 +433,10 @@ void XdsOverrideHostLb::ResetBackoffLocked() {
 
 absl::Status XdsOverrideHostLb::UpdateLocked(UpdateArgs args) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) {
-    gpr_log(GPR_INFO, "[xds_override_host_lb %p] Received update", this);
+    gpr_log(GPR_INFO,
+            "[xds_override_host_lb %p] Received update with %" PRIuPTR
+            " addresses",
+            this, args.addresses.ok() ? args.addresses->size() : 0);
   }
   auto old_config = std::move(config_);
   // Update config.
@@ -498,6 +504,10 @@ OrphanablePtr<LoadBalancingPolicy> XdsOverrideHostLb::CreateChildPolicyLocked(
 absl::StatusOr<ServerAddressList> XdsOverrideHostLb::UpdateAddressMap(
     absl::StatusOr<ServerAddressList> addresses) {
   if (!addresses.ok()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) {
+      gpr_log(GPR_INFO, "[xds_override_host_lb %p] address error: %s", this,
+              addresses.status().ToString().c_str());
+    }
     return addresses;
   }
   ServerAddressList return_value;
@@ -505,13 +515,30 @@ absl::StatusOr<ServerAddressList> XdsOverrideHostLb::UpdateAddressMap(
   for (const auto& address : *addresses) {
     XdsHealthStatus status = GetAddressHealthStatus(address);
     if (status.status() != XdsHealthStatus::kDraining) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) {
+        gpr_log(GPR_INFO,
+                "[xds_override_host_lb %p] address %s: not draining, "
+                "passing to child",
+                this, address.ToString().c_str());
+      }
       return_value.push_back(address);
     } else if (!config_->override_host_status_set().Contains(status)) {
       // Skip draining hosts if not in the override status set.
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) {
+        gpr_log(GPR_INFO,
+                "[xds_override_host_lb %p] address %s: draining but not in "
+                "override_host_status set -- ignoring",
+                this, address.ToString().c_str());
+      }
       continue;
     }
     auto key = grpc_sockaddr_to_uri(&address.address());
     if (key.ok()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) {
+        gpr_log(GPR_INFO,
+                "[xds_override_host_lb %p] address %s: adding map key %s", this,
+                address.ToString().c_str(), key->c_str());
+      }
       addresses_for_map.emplace(std::move(*key), status);
     }
   }
@@ -519,6 +546,10 @@ absl::StatusOr<ServerAddressList> XdsOverrideHostLb::UpdateAddressMap(
     MutexLock lock(&subchannel_map_mu_);
     for (auto it = subchannel_map_.begin(); it != subchannel_map_.end();) {
       if (addresses_for_map.find(it->first) == addresses_for_map.end()) {
+        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) {
+          gpr_log(GPR_INFO, "[xds_override_host_lb %p] removing map key %s",
+                  this, it->first.c_str());
+        }
         it = subchannel_map_.erase(it);
       } else {
         ++it;
@@ -527,10 +558,20 @@ absl::StatusOr<ServerAddressList> XdsOverrideHostLb::UpdateAddressMap(
     for (const auto& key_status : addresses_for_map) {
       auto it = subchannel_map_.find(key_status.first);
       if (it == subchannel_map_.end()) {
+        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) {
+          gpr_log(GPR_INFO, "[xds_override_host_lb %p] adding map key %s", this,
+                  key_status.first.c_str());
+        }
         subchannel_map_.emplace(std::piecewise_construct,
                                 std::forward_as_tuple(key_status.first),
                                 std::forward_as_tuple(key_status.second));
       } else {
+        if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) {
+          gpr_log(GPR_INFO,
+                  "[xds_override_host_lb %p] setting EDS health status for "
+                  "%s to %s",
+                  this, key_status.first.c_str(), key_status.second.ToString());
+        }
         it->second.SetEdsHealthStatus(key_status.second);
       }
     }
@@ -678,8 +719,19 @@ void XdsOverrideHostLb::SubchannelWrapper::UpdateConnectivityState(
 }
 
 void XdsOverrideHostLb::SubchannelWrapper::Orphan() {
-  key_.reset();
-  wrapped_subchannel()->CancelConnectivityStateWatch(watcher_);
+  if (!IsWorkSerializerDispatchEnabled()) {
+    key_.reset();
+    wrapped_subchannel()->CancelConnectivityStateWatch(watcher_);
+    return;
+  }
+  WeakRefCountedPtr<SubchannelWrapper> self = WeakRef();
+  policy_->work_serializer()->Run(
+      [self = std::move(self)]() {
+        self->key_.reset();
+        self->wrapped_subchannel()->CancelConnectivityStateWatch(
+            self->watcher_);
+      },
+      DEBUG_LOCATION);
 }
 
 grpc_pollset_set* XdsOverrideHostLb::SubchannelWrapper::