grpc 1.58.0 → 1.59.0

data/src/core/ext/filters/client_channel/lb_policy/health_check_client.cc

@@ -28,6 +28,7 @@
 
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 #include "upb/base/string_view.h"
@@ -44,6 +45,7 @@
 #include "src/core/ext/filters/client_channel/lb_policy/health_check_client_internal.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
 #include "src/core/ext/filters/client_channel/subchannel_stream_client.h"
+#include "src/core/lib/address_utils/sockaddr_utils.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_trace.h"
 #include "src/core/lib/debug/trace.h"
@@ -114,7 +116,7 @@ void HealthProducer::HealthChecker::Orphan() {
 
 void HealthProducer::HealthChecker::AddWatcherLocked(HealthWatcher* watcher) {
   watchers_.insert(watcher);
-  watcher->Notify(state_, status_);
+  if (state_.has_value()) watcher->Notify(*state_, status_);
 }
 
 bool HealthProducer::HealthChecker::RemoveWatcherLocked(
@@ -128,13 +130,18 @@ void HealthProducer::HealthChecker::OnConnectivityStateChangeLocked(
   if (state == GRPC_CHANNEL_READY) {
     // We should already be in CONNECTING, and we don't want to change
     // that until we see the initial response on the stream.
-    GPR_ASSERT(state_ == GRPC_CHANNEL_CONNECTING);
+    if (!state_.has_value()) {
+      state_ = GRPC_CHANNEL_CONNECTING;
+      status_ = absl::OkStatus();
+    } else {
+      GPR_ASSERT(state_ == GRPC_CHANNEL_CONNECTING);
+    }
     // Start the health watch stream.
     StartHealthStreamLocked();
   } else {
     state_ = state;
     status_ = status;
-    NotifyWatchersLocked(state_, status_);
+    NotifyWatchersLocked(*state_, status_);
     // We're not connected, so stop health checking.
     stream_client_.reset();
   }
@@ -177,12 +184,21 @@ void HealthProducer::HealthChecker::NotifyWatchersLocked(
 void HealthProducer::HealthChecker::OnHealthWatchStatusChange(
     grpc_connectivity_state state, const absl::Status& status) {
   if (state == GRPC_CHANNEL_SHUTDOWN) return;
+  // Prepend the subchannel's address to the status if needed.
+  absl::Status use_status;
+  if (!status.ok()) {
+    std::string address_str =
+        grpc_sockaddr_to_uri(&producer_->subchannel_->address())
+            .value_or("<unknown address type>");
+    use_status = absl::Status(
+        status.code(), absl::StrCat(address_str, ": ", status.message()));
+  }
   work_serializer_->Schedule(
-      [self = Ref(), state, status]() {
+      [self = Ref(), state, status = std::move(use_status)]() mutable {
         MutexLock lock(&self->producer_->mu_);
         if (self->stream_client_ != nullptr) {
           self->state_ = state;
-          self->status_ = status;
+          self->status_ = std::move(status);
           for (HealthWatcher* watcher : self->watchers_) {
             watcher->Notify(state, self->status_);
           }
@@ -364,7 +380,7 @@ void HealthProducer::AddWatcher(
   grpc_pollset_set_add_pollset_set(interested_parties_,
                                    watcher->interested_parties());
   if (!health_check_service_name.has_value()) {
-    watcher->Notify(state_, status_);
+    if (state_.has_value()) watcher->Notify(*state_, status_);
     non_health_watchers_.insert(watcher);
   } else {
     auto it =
@@ -421,6 +437,13 @@ void HealthProducer::OnConnectivityStateChange(grpc_connectivity_state state,
 //
 
 HealthWatcher::~HealthWatcher() {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
+    gpr_log(GPR_INFO,
+            "HealthWatcher %p: unregistering from producer %p "
+            "(health_check_service_name=\"%s\")",
+            this, producer_.get(),
+            health_check_service_name_.value_or("N/A").c_str());
+  }
   if (producer_ != nullptr) {
     producer_->RemoveWatcher(this, health_check_service_name_);
   }
@@ -447,6 +470,13 @@ void HealthWatcher::SetSubchannel(Subchannel* subchannel) {
   if (created) producer_->Start(subchannel->Ref());
   // Register ourself with the producer.
   producer_->AddWatcher(this, health_check_service_name_);
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
+    gpr_log(GPR_INFO,
+            "HealthWatcher %p: registered with producer %p (created=%d, "
+            "health_check_service_name=\"%s\")",
+            this, producer_.get(), created,
+            health_check_service_name_.value_or("N/A").c_str());
+  }
 }
 
 void HealthWatcher::Notify(grpc_connectivity_state state, absl::Status status) {
@@ -472,6 +502,11 @@ MakeHealthCheckWatcher(
     health_check_service_name =
         args.GetOwnedString(GRPC_ARG_HEALTH_CHECK_SERVICE_NAME);
   }
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
+    gpr_log(GPR_INFO,
+            "creating HealthWatcher -- health_check_service_name=\"%s\"",
+            health_check_service_name.value_or("N/A").c_str());
+  }
   return std::make_unique<HealthWatcher>(std::move(work_serializer),
                                          std::move(health_check_service_name),
                                          std::move(watcher));

data/src/core/ext/filters/client_channel/lb_policy/health_check_client_internal.h

@@ -125,9 +125,11 @@ class HealthProducer : public Subchannel::DataProducerInterface {
     WeakRefCountedPtr<HealthProducer> producer_;
     absl::string_view health_check_service_name_;
     std::shared_ptr<WorkSerializer> work_serializer_ =
-        std::make_shared<WorkSerializer>();
+        std::make_shared<WorkSerializer>(
+            producer_->subchannel_->event_engine());
 
-    grpc_connectivity_state state_ ABSL_GUARDED_BY(&HealthProducer::mu_);
+    absl::optional<grpc_connectivity_state> state_
+        ABSL_GUARDED_BY(&HealthProducer::mu_);
     absl::Status status_ ABSL_GUARDED_BY(&HealthProducer::mu_);
     OrphanablePtr<SubchannelStreamClient> stream_client_
         ABSL_GUARDED_BY(&HealthProducer::mu_);
@@ -143,7 +145,7 @@ class HealthProducer : public Subchannel::DataProducerInterface {
   grpc_pollset_set* interested_parties_;
 
   Mutex mu_;
-  grpc_connectivity_state state_ ABSL_GUARDED_BY(&mu_);
+  absl::optional<grpc_connectivity_state> state_ ABSL_GUARDED_BY(&mu_);
   absl::Status status_ ABSL_GUARDED_BY(&mu_);
   RefCountedPtr<ConnectedSubchannel> connected_subchannel_
       ABSL_GUARDED_BY(&mu_);

data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc

@@ -49,8 +49,8 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/gprpp/debug_location.h"
-#include "src/core/lib/gprpp/match.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
@@ -124,13 +124,12 @@ class OutlierDetectionLb : public LoadBalancingPolicy {
   class SubchannelState;
   class SubchannelWrapper : public DelegatingSubchannel {
    public:
-    SubchannelWrapper(RefCountedPtr<SubchannelState> subchannel_state,
-                      RefCountedPtr<SubchannelInterface> subchannel,
-                      bool disable_via_raw_connectivity_watch)
+    SubchannelWrapper(std::shared_ptr<WorkSerializer> work_serializer,
+                      RefCountedPtr<SubchannelState> subchannel_state,
+                      RefCountedPtr<SubchannelInterface> subchannel)
         : DelegatingSubchannel(std::move(subchannel)),
-          subchannel_state_(std::move(subchannel_state)),
-          disable_via_raw_connectivity_watch_(
-              disable_via_raw_connectivity_watch) {
+          work_serializer_(std::move(work_serializer)),
+          subchannel_state_(std::move(subchannel_state)) {
       if (subchannel_state_ != nullptr) {
         subchannel_state_->AddSubchannel(this);
         if (subchannel_state_->ejection_time().has_value()) {
@@ -139,22 +138,27 @@ class OutlierDetectionLb : public LoadBalancingPolicy {
       }
     }
 
-    ~SubchannelWrapper() override {
-      if (subchannel_state_ != nullptr) {
-        subchannel_state_->RemoveSubchannel(this);
+    void Orphan() override {
+      if (!IsWorkSerializerDispatchEnabled()) {
+        if (subchannel_state_ != nullptr) {
+          subchannel_state_->RemoveSubchannel(this);
+        }
+        return;
       }
+      WeakRefCountedPtr<SubchannelWrapper> self = WeakRef();
+      work_serializer_->Run(
+          [self = std::move(self)]() {
+            if (self->subchannel_state_ != nullptr) {
+              self->subchannel_state_->RemoveSubchannel(self.get());
+            }
+          },
+          DEBUG_LOCATION);
     }
 
     void Eject();
 
     void Uneject();
 
-    void WatchConnectivityState(
-        std::unique_ptr<ConnectivityStateWatcherInterface> watcher) override;
-
-    void CancelConnectivityStateWatch(
-        ConnectivityStateWatcherInterface* watcher) override;
-
     void AddDataWatcher(std::unique_ptr<DataWatcherInterface> watcher) override;
 
     RefCountedPtr<SubchannelState> subchannel_state() const {
@@ -162,11 +166,6 @@ class OutlierDetectionLb : public LoadBalancingPolicy {
     }
 
    private:
-    // TODO(roth): As a temporary hack, this needs to handle watchers
-    // stored as both unique_ptr<> and shared_ptr<>, since the former is
-    // used for raw connectivity state watches and the latter is used
-    // for health watches.  This hack will go away as part of implementing
-    // dualstack backend support.
     class WatcherWrapper
         : public SubchannelInterface::ConnectivityStateWatcherInterface {
      public:
@@ -176,16 +175,10 @@ class OutlierDetectionLb : public LoadBalancingPolicy {
                      bool ejected)
           : watcher_(std::move(health_watcher)), ejected_(ejected) {}
 
-      WatcherWrapper(std::unique_ptr<
-                         SubchannelInterface::ConnectivityStateWatcherInterface>
-                         watcher,
-                     bool ejected)
-          : watcher_(std::move(watcher)), ejected_(ejected) {}
-
       void Eject() {
         ejected_ = true;
         if (last_seen_state_.has_value()) {
-          watcher()->OnConnectivityStateChange(
+          watcher_->OnConnectivityStateChange(
               GRPC_CHANNEL_TRANSIENT_FAILURE,
               absl::UnavailableError(
                   "subchannel ejected by outlier detection"));
@@ -195,8 +188,8 @@ class OutlierDetectionLb : public LoadBalancingPolicy {
       void Uneject() {
         ejected_ = false;
         if (last_seen_state_.has_value()) {
-          watcher()->OnConnectivityStateChange(*last_seen_state_,
-                                               last_seen_status_);
+          watcher_->OnConnectivityStateChange(*last_seen_state_,
+                                              last_seen_status_);
         }
       }
 
@@ -211,43 +204,26 @@ class OutlierDetectionLb : public LoadBalancingPolicy {
             status = absl::UnavailableError(
                 "subchannel ejected by outlier detection");
           }
-          watcher()->OnConnectivityStateChange(new_state, status);
+          watcher_->OnConnectivityStateChange(new_state, status);
         }
       }
 
       grpc_pollset_set* interested_parties() override {
-        return watcher()->interested_parties();
+        return watcher_->interested_parties();
       }
 
      private:
-      SubchannelInterface::ConnectivityStateWatcherInterface* watcher() const {
-        return Match(
-            watcher_,
-            [](const std::shared_ptr<
-                SubchannelInterface::ConnectivityStateWatcherInterface>&
-                   watcher) { return watcher.get(); },
-            [](const std::unique_ptr<
-                SubchannelInterface::ConnectivityStateWatcherInterface>&
-                   watcher) { return watcher.get(); });
-      }
-
-      absl::variant<std::shared_ptr<
-                        SubchannelInterface::ConnectivityStateWatcherInterface>,
-                    std::unique_ptr<
-                        SubchannelInterface::ConnectivityStateWatcherInterface>>
+      std::shared_ptr<SubchannelInterface::ConnectivityStateWatcherInterface>
           watcher_;
       absl::optional<grpc_connectivity_state> last_seen_state_;
       absl::Status last_seen_status_;
       bool ejected_;
     };
 
+    std::shared_ptr<WorkSerializer> work_serializer_;
     RefCountedPtr<SubchannelState> subchannel_state_;
-    const bool disable_via_raw_connectivity_watch_;
     bool ejected_ = false;
-    std::map<SubchannelInterface::ConnectivityStateWatcherInterface*,
-             WatcherWrapper*>
-        watchers_;
-    WatcherWrapper* watcher_wrapper_ = nullptr;  // For health watching.
+    WatcherWrapper* watcher_wrapper_ = nullptr;
   };
 
   class SubchannelState : public RefCounted<SubchannelState> {
@@ -428,50 +404,14 @@ class OutlierDetectionLb : public LoadBalancingPolicy {
 
 void OutlierDetectionLb::SubchannelWrapper::Eject() {
   ejected_ = true;
-  // Ejecting the subchannel may cause the child policy to cancel the watch,
-  // so we need to be prepared for the map to be modified while we are
-  // iterating.
-  for (auto it = watchers_.begin(); it != watchers_.end();) {
-    WatcherWrapper* watcher = it->second;
-    ++it;
-    watcher->Eject();
-  }
   if (watcher_wrapper_ != nullptr) watcher_wrapper_->Eject();
 }
 
 void OutlierDetectionLb::SubchannelWrapper::Uneject() {
   ejected_ = false;
-  for (auto& watcher : watchers_) {
-    watcher.second->Uneject();
-  }
   if (watcher_wrapper_ != nullptr) watcher_wrapper_->Uneject();
 }
 
-void OutlierDetectionLb::SubchannelWrapper::WatchConnectivityState(
-    std::unique_ptr<ConnectivityStateWatcherInterface> watcher) {
-  if (disable_via_raw_connectivity_watch_) {
-    wrapped_subchannel()->WatchConnectivityState(std::move(watcher));
-    return;
-  }
-  ConnectivityStateWatcherInterface* watcher_ptr = watcher.get();
-  auto watcher_wrapper =
-      std::make_unique<WatcherWrapper>(std::move(watcher), ejected_);
-  watchers_.emplace(watcher_ptr, watcher_wrapper.get());
-  wrapped_subchannel()->WatchConnectivityState(std::move(watcher_wrapper));
-}
-
-void OutlierDetectionLb::SubchannelWrapper::CancelConnectivityStateWatch(
-    ConnectivityStateWatcherInterface* watcher) {
-  if (disable_via_raw_connectivity_watch_) {
-    wrapped_subchannel()->CancelConnectivityStateWatch(watcher);
-    return;
-  }
-  auto it = watchers_.find(watcher);
-  if (it == watchers_.end()) return;
-  wrapped_subchannel()->CancelConnectivityStateWatch(it->second);
-  watchers_.erase(it);
-}
-
 void OutlierDetectionLb::SubchannelWrapper::AddDataWatcher(
     std::unique_ptr<DataWatcherInterface> watcher) {
   auto* w = static_cast<InternalSubchannelDataWatcherInterface*>(watcher.get());
@@ -777,22 +717,12 @@ OrphanablePtr<LoadBalancingPolicy> OutlierDetectionLb::CreateChildPolicyLocked(
 RefCountedPtr<SubchannelInterface> OutlierDetectionLb::Helper::CreateSubchannel(
     ServerAddress address, const ChannelArgs& args) {
   if (parent()->shutting_down_) return nullptr;
-  // If the address has the DisableOutlierDetectionAttribute attribute,
-  // ignore it for raw connectivity state updates.
-  // TODO(roth): This is a hack to prevent outlier_detection from
-  // working with pick_first, as per discussion in
-  // https://github.com/grpc/grpc/issues/32967.  Remove this as part of
-  // implementing dualstack backend support.
-  const bool disable_via_raw_connectivity_watch =
-      address.args().GetInt(GRPC_ARG_OUTLIER_DETECTION_DISABLE) == 1;
   RefCountedPtr<SubchannelState> subchannel_state;
   std::string key = MakeKeyForAddress(address);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_outlier_detection_lb_trace)) {
     gpr_log(GPR_INFO,
-            "[outlier_detection_lb %p] using key %s for subchannel "
-            "address %s, disable_via_raw_connectivity_watch=%d",
-            parent(), key.c_str(), address.ToString().c_str(),
-            disable_via_raw_connectivity_watch);
+            "[outlier_detection_lb %p] using key %s for subchannel address %s",
+            parent(), key.c_str(), address.ToString().c_str());
   }
   if (!key.empty()) {
     auto it = parent()->subchannel_state_map_.find(key);
@@ -801,10 +731,9 @@ RefCountedPtr<SubchannelInterface> OutlierDetectionLb::Helper::CreateSubchannel(
     }
   }
   auto subchannel = MakeRefCounted<SubchannelWrapper>(
-      subchannel_state,
+      parent()->work_serializer(), subchannel_state,
       parent()->channel_control_helper()->CreateSubchannel(std::move(address),
-                                                           args),
-      disable_via_raw_connectivity_watch);
+                                                           args));
   if (subchannel_state != nullptr) {
     subchannel_state->AddSubchannel(subchannel.get());
   }

data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.h

@@ -28,7 +28,6 @@
 #include "src/core/lib/json/json.h"
 #include "src/core/lib/json/json_args.h"
 #include "src/core/lib/json/json_object_loader.h"
-#include "src/core/lib/resolver/server_address.h"
 
 namespace grpc_core {
 
@@ -90,12 +89,6 @@ struct OutlierDetectionConfig {
                     ValidationErrors* errors);
 };
 
-// TODO(roth): This is a horrible hack used to disable outlier detection
-// when used with the pick_first policy.  Remove this as part of
-// implementing the dualstack backend design.
-#define GRPC_ARG_OUTLIER_DETECTION_DISABLE \
-  GRPC_ARG_NO_SUBCHANNEL_PREFIX "outlier_detection_disable"
-
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_OUTLIER_DETECTION_OUTLIER_DETECTION_H