grpc 1.46.3 → 1.47.0

data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc

@@ -16,21 +16,36 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <inttypes.h>
 #include <stdlib.h>
-#include <string.h>
 
-#include <grpc/support/alloc.h>
+#include <memory>
+#include <string>
+#include <utility>
 
+#include "absl/container/inlined_vector.h"
+#include "absl/memory/memory.h"
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "absl/strings/str_cat.h"
+
+#include <grpc/impl/codegen/connectivity_state.h>
+#include <grpc/impl/codegen/grpc_types.h>
+#include <grpc/support/log.h>
+
+#include "src/core/ext/filters/client_channel/lb_policy.h"
 #include "src/core/ext/filters/client_channel/lb_policy/subchannel_list.h"
+#include "src/core/ext/filters/client_channel/lb_policy_factory.h"
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
-#include "src/core/ext/filters/client_channel/subchannel.h"
-#include "src/core/lib/address_utils/sockaddr_utils.h"
-#include "src/core/lib/channel/channel_args.h"
+#include "src/core/ext/filters/client_channel/subchannel_interface.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
-#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/resolver/server_address.h"
 #include "src/core/lib/transport/connectivity_state.h"
-#include "src/core/lib/transport/error_utils.h"
 
 namespace grpc_core {
 
@@ -75,14 +90,15 @@ class RoundRobin : public LoadBalancingPolicy {
         : SubchannelData(subchannel_list, address, std::move(subchannel)) {}
 
     grpc_connectivity_state connectivity_state() const {
-      return last_connectivity_state_;
+      return logical_connectivity_state_;
     }
 
-    bool seen_failure_since_ready() const { return seen_failure_since_ready_; }
-
-    // Performs connectivity state updates that need to be done both when we
-    // first start watching and when a watcher notification is received.
-    void UpdateConnectivityStateLocked(
+    // Computes and updates the logical connectivity state of the subchannel.
+    // Note that the logical connectivity state may differ from the
+    // actual reported state in some cases (e.g., after we see
+    // TRANSIENT_FAILURE, we ignore any subsequent state changes until
+    // we see READY).  Returns true if the state changed.
+    bool UpdateLogicalConnectivityStateLocked(
         grpc_connectivity_state connectivity_state);
 
    private:
@@ -91,8 +107,7 @@ class RoundRobin : public LoadBalancingPolicy {
     void ProcessConnectivityChangeLocked(
         grpc_connectivity_state connectivity_state) override;
 
-    grpc_connectivity_state last_connectivity_state_ = GRPC_CHANNEL_IDLE;
-    bool seen_failure_since_ready_ = false;
+    grpc_connectivity_state logical_connectivity_state_ = GRPC_CHANNEL_IDLE;
   };
 
   // A list of subchannels.
@@ -100,11 +115,14 @@ class RoundRobin : public LoadBalancingPolicy {
       : public SubchannelList<RoundRobinSubchannelList,
                               RoundRobinSubchannelData> {
    public:
-    RoundRobinSubchannelList(RoundRobin* policy, TraceFlag* tracer,
-                             ServerAddressList addresses,
+    RoundRobinSubchannelList(RoundRobin* policy, ServerAddressList addresses,
                              const grpc_channel_args& args)
-        : SubchannelList(policy, tracer, std::move(addresses),
-                         policy->channel_control_helper(), args) {
+        : SubchannelList(policy,
+                         (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)
+                              ? "RoundRobinSubchannelList"
+                              : nullptr),
+                         std::move(addresses), policy->channel_control_helper(),
+                         args) {
       // Need to maintain a ref to the LB policy as long as we maintain
       // any references to subchannels, since the subchannels'
       // pollset_sets will include the LB policy's pollset_set.
@@ -117,23 +135,27 @@ class RoundRobin : public LoadBalancingPolicy {
     }
 
     // Starts watching the subchannels in this list.
-    void StartWatchingLocked();
+    void StartWatchingLocked(absl::Status status_for_tf);
 
     // Updates the counters of subchannels in each state when a
     // subchannel transitions from old_state to new_state.
     void UpdateStateCountersLocked(grpc_connectivity_state old_state,
                                    grpc_connectivity_state new_state);
 
-    // If this subchannel list is the RR policy's current subchannel
-    // list, updates the RR policy's connectivity state based on the
-    // subchannel list's state counters.
-    void MaybeUpdateRoundRobinConnectivityStateLocked();
-
-    // Updates the RR policy's overall state based on the counters of
-    // subchannels in each state.
-    void UpdateRoundRobinStateFromSubchannelStateCountsLocked();
+    // Ensures that the right subchannel list is used and then updates
+    // the RR policy's connectivity state based on the subchannel list's
+    // state counters.
+    void MaybeUpdateRoundRobinConnectivityStateLocked(
+        absl::Status status_for_tf);
 
    private:
+    std::string CountersString() const {
+      return absl::StrCat("num_subchannels=", num_subchannels(),
+                          " num_ready=", num_ready_,
+                          " num_connecting=", num_connecting_,
+                          " num_transient_failure=", num_transient_failure_);
+    }
+
     size_t num_ready_ = 0;
     size_t num_connecting_ = 0;
     size_t num_transient_failure_ = 0;
@@ -238,8 +260,45 @@ void RoundRobin::ResetBackoffLocked() {
   }
 }
 
-void RoundRobin::RoundRobinSubchannelList::StartWatchingLocked() {
-  if (num_subchannels() == 0) return;
+void RoundRobin::UpdateLocked(UpdateArgs args) {
+  ServerAddressList addresses;
+  if (args.addresses.ok()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
+      gpr_log(GPR_INFO, "[RR %p] received update with %" PRIuPTR " addresses",
+              this, args.addresses->size());
+    }
+    addresses = std::move(*args.addresses);
+  } else {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
+      gpr_log(GPR_INFO, "[RR %p] received update with address error: %s", this,
+              args.addresses.status().ToString().c_str());
+    }
+    // If we already have a subchannel list, then ignore the resolver
+    // failure and keep using the existing list.
+    if (subchannel_list_ != nullptr) return;
+  }
+  // Create new subchannel list, replacing the previous pending list, if any.
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace) &&
+      latest_pending_subchannel_list_ != nullptr) {
+    gpr_log(GPR_INFO, "[RR %p] replacing previous pending subchannel list %p",
+            this, latest_pending_subchannel_list_.get());
+  }
+  latest_pending_subchannel_list_ = MakeOrphanable<RoundRobinSubchannelList>(
+      this, std::move(addresses), *args.args);
+  // Start watching the new list.  If appropriate, this will cause it to be
+  // immediately promoted to subchannel_list_ and to generate a new picker.
+  latest_pending_subchannel_list_->StartWatchingLocked(
+      args.addresses.ok() ? absl::UnavailableError(absl::StrCat(
+                                "empty address list: ", args.resolution_note))
+                          : args.addresses.status());
+}
+
+//
+// RoundRobinSubchannelList
+//
+
+void RoundRobin::RoundRobinSubchannelList::StartWatchingLocked(
+    absl::Status status_for_tf) {
   // Check current state of each subchannel synchronously, since any
   // subchannel already used by some other channel may have a non-IDLE
   // state.
@@ -247,18 +306,18 @@ void RoundRobin::RoundRobinSubchannelList::StartWatchingLocked() {
     grpc_connectivity_state state =
         subchannel(i)->CheckConnectivityStateLocked();
     if (state != GRPC_CHANNEL_IDLE) {
-      subchannel(i)->UpdateConnectivityStateLocked(state);
+      subchannel(i)->UpdateLogicalConnectivityStateLocked(state);
     }
   }
   // Start connectivity watch for each subchannel.
   for (size_t i = 0; i < num_subchannels(); i++) {
     if (subchannel(i)->subchannel() != nullptr) {
       subchannel(i)->StartConnectivityWatchLocked();
-      subchannel(i)->subchannel()->AttemptToConnect();
+      subchannel(i)->subchannel()->RequestConnection();
     }
   }
-  // Now set the LB policy's state based on the subchannels' states.
-  UpdateRoundRobinStateFromSubchannelStateCountsLocked();
+  // Update RR connectivity state if needed.
+  MaybeUpdateRoundRobinConnectivityStateLocked(status_for_tf);
 }
 
 void RoundRobin::RoundRobinSubchannelList::UpdateStateCountersLocked(
@@ -284,80 +343,73 @@ void RoundRobin::RoundRobinSubchannelList::UpdateStateCountersLocked(
   }
 }
 
-// Sets the RR policy's connectivity state and generates a new picker based
-// on the current subchannel list.
 void RoundRobin::RoundRobinSubchannelList::
-    MaybeUpdateRoundRobinConnectivityStateLocked() {
+    MaybeUpdateRoundRobinConnectivityStateLocked(absl::Status status_for_tf) {
   RoundRobin* p = static_cast<RoundRobin*>(policy());
+  // If this is latest_pending_subchannel_list_, then swap it into
+  // subchannel_list_ in the following cases:
+  // - subchannel_list_ is null (i.e., this is the first update).
+  // - subchannel_list_ has no READY subchannels.
+  // - This list has at least one READY subchannel.
+  // - All of the subchannels in this list are in TRANSIENT_FAILURE, or
+  //   the list is empty.  (This may cause the channel to go from READY
+  //   to TRANSIENT_FAILURE, but we're doing what the control plane told
+  //   us to do.
+  if (p->latest_pending_subchannel_list_.get() == this &&
+      (p->subchannel_list_ == nullptr || p->subchannel_list_->num_ready_ == 0 ||
+       num_ready_ > 0 ||
+       // Note: num_transient_failure_ and num_subchannels() may both be 0.
+       num_transient_failure_ == num_subchannels())) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
+      const std::string old_counters_string =
+          p->subchannel_list_ != nullptr ? p->subchannel_list_->CountersString()
+                                         : "";
+      gpr_log(
+          GPR_INFO,
+          "[RR %p] swapping out subchannel list %p (%s) in favor of %p (%s)", p,
+          p->subchannel_list_.get(), old_counters_string.c_str(), this,
+          CountersString().c_str());
+    }
+    p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
+  }
   // Only set connectivity state if this is the current subchannel list.
   if (p->subchannel_list_.get() != this) return;
-  // In priority order. The first rule to match terminates the search (ie, if we
-  // are on rule n, all previous rules were unfulfilled).
-  //
-  // 1) RULE: ANY subchannel is READY => policy is READY.
-  //    CHECK: subchannel_list->num_ready > 0.
-  //
-  // 2) RULE: ANY subchannel is CONNECTING => policy is CONNECTING.
-  //    CHECK: sd->curr_connectivity_state == CONNECTING.
-  //
-  // 3) RULE: ALL subchannels are TRANSIENT_FAILURE => policy is
-  //                                                   TRANSIENT_FAILURE.
-  //    CHECK: subchannel_list->num_transient_failures ==
-  //           subchannel_list->num_subchannels.
+  // First matching rule wins:
+  // 1) ANY subchannel is READY => policy is READY.
+  // 2) ANY subchannel is CONNECTING => policy is CONNECTING.
+  // 3) ALL subchannels are TRANSIENT_FAILURE => policy is TRANSIENT_FAILURE.
   if (num_ready_ > 0) {
-    // 1) READY
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
+      gpr_log(GPR_INFO, "[RR %p] reporting READY with subchannel list %p", p,
+              this);
+    }
     p->channel_control_helper()->UpdateState(
         GRPC_CHANNEL_READY, absl::Status(), absl::make_unique<Picker>(p, this));
   } else if (num_connecting_ > 0) {
-    // 2) CONNECTING
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
+      gpr_log(GPR_INFO, "[RR %p] reporting CONNECTING with subchannel list %p",
+              p, this);
+    }
     p->channel_control_helper()->UpdateState(
         GRPC_CHANNEL_CONNECTING, absl::Status(),
         absl::make_unique<QueuePicker>(p->Ref(DEBUG_LOCATION, "QueuePicker")));
   } else if (num_transient_failure_ == num_subchannels()) {
-    // 3) TRANSIENT_FAILURE
-    absl::Status status =
-        absl::UnavailableError("connections to all backends failing");
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
+      gpr_log(GPR_INFO,
+              "[RR %p] reporting TRANSIENT_FAILURE with subchannel list %p: %s",
+              p, this, status_for_tf.ToString().c_str());
+    }
     p->channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_TRANSIENT_FAILURE, status,
-        absl::make_unique<TransientFailurePicker>(status));
+        GRPC_CHANNEL_TRANSIENT_FAILURE, status_for_tf,
+        absl::make_unique<TransientFailurePicker>(status_for_tf));
   }
 }
 
-void RoundRobin::RoundRobinSubchannelList::
-    UpdateRoundRobinStateFromSubchannelStateCountsLocked() {
-  RoundRobin* p = static_cast<RoundRobin*>(policy());
-  // If we have at least one READY subchannel, then swap to the new list.
-  // Also, if all of the subchannels are in TRANSIENT_FAILURE, then we know
-  // we've tried all of them and failed, so we go ahead and swap over
-  // anyway; this may cause the channel to go from READY to TRANSIENT_FAILURE,
-  // but we are doing what the control plane told us to do.
-  if (num_ready_ > 0 || num_transient_failure_ == num_subchannels()) {
-    if (p->subchannel_list_.get() != this) {
-      // Promote this list to p->subchannel_list_.
-      // This list must be p->latest_pending_subchannel_list_, because
-      // any previous update would have been shut down already and
-      // therefore we would not be receiving a notification for them.
-      GPR_ASSERT(p->latest_pending_subchannel_list_.get() == this);
-      GPR_ASSERT(!shutting_down());
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
-        const size_t old_num_subchannels =
-            p->subchannel_list_ != nullptr
-                ? p->subchannel_list_->num_subchannels()
-                : 0;
-        gpr_log(GPR_INFO,
-                "[RR %p] phasing out subchannel list %p (size %" PRIuPTR
-                ") in favor of %p (size %" PRIuPTR ")",
-                p, p->subchannel_list_.get(), old_num_subchannels, this,
-                num_subchannels());
-      }
-      p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
-    }
-  }
-  // Update the RR policy's connectivity state if needed.
-  MaybeUpdateRoundRobinConnectivityStateLocked();
-}
+//
+// RoundRobinSubchannelData
+//
 
-void RoundRobin::RoundRobinSubchannelData::UpdateConnectivityStateLocked(
+bool RoundRobin::RoundRobinSubchannelData::UpdateLogicalConnectivityStateLocked(
     grpc_connectivity_state connectivity_state) {
   RoundRobin* p = static_cast<RoundRobin*>(subchannel_list()->policy());
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
@@ -367,115 +419,74 @@ void RoundRobin::RoundRobinSubchannelData::UpdateConnectivityStateLocked(
         "(index %" PRIuPTR " of %" PRIuPTR "): prev_state=%s new_state=%s",
         p, subchannel(), subchannel_list(), Index(),
         subchannel_list()->num_subchannels(),
-        ConnectivityStateName(last_connectivity_state_),
+        ConnectivityStateName(logical_connectivity_state_),
         ConnectivityStateName(connectivity_state));
   }
   // Decide what state to report for aggregation purposes.
-  // If we haven't seen a failure since the last time we were in state
-  // READY, then we report the state change as-is.  However, once we do see
-  // a failure, we report TRANSIENT_FAILURE and do not report any subsequent
-  // state changes until we go back into state READY.
-  if (!seen_failure_since_ready_) {
-    if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-      seen_failure_since_ready_ = true;
-    }
-    subchannel_list()->UpdateStateCountersLocked(last_connectivity_state_,
-                                                 connectivity_state);
-  } else {
-    if (connectivity_state == GRPC_CHANNEL_READY) {
-      seen_failure_since_ready_ = false;
-      subchannel_list()->UpdateStateCountersLocked(
-          GRPC_CHANNEL_TRANSIENT_FAILURE, connectivity_state);
+  // If the last logical state was TRANSIENT_FAILURE, then ignore the
+  // state change unless the new state is READY.
+  if (logical_connectivity_state_ == GRPC_CHANNEL_TRANSIENT_FAILURE &&
+      connectivity_state != GRPC_CHANNEL_READY) {
+    return false;
+  }
+  // If the new state is IDLE, treat it as CONNECTING, since it will
+  // immediately transition into CONNECTING anyway.
+  if (connectivity_state == GRPC_CHANNEL_IDLE) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
+      gpr_log(GPR_INFO,
+              "[RR %p] subchannel %p, subchannel_list %p (index %" PRIuPTR
+              " of %" PRIuPTR "): treating IDLE as CONNECTING",
+              p, subchannel(), subchannel_list(), Index(),
+              subchannel_list()->num_subchannels());
     }
+    connectivity_state = GRPC_CHANNEL_CONNECTING;
   }
-  // Record last seen connectivity state.
-  last_connectivity_state_ = connectivity_state;
+  // If no change, return false.
+  if (logical_connectivity_state_ == connectivity_state) return false;
+  // Otherwise, update counters and logical state.
+  subchannel_list()->UpdateStateCountersLocked(logical_connectivity_state_,
+                                               connectivity_state);
+  logical_connectivity_state_ = connectivity_state;
+  return true;
 }
 
 void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
     grpc_connectivity_state connectivity_state) {
   RoundRobin* p = static_cast<RoundRobin*>(subchannel_list()->policy());
   GPR_ASSERT(subchannel() != nullptr);
-  // If the new state is TRANSIENT_FAILURE, re-resolve.
+  // If the new state is TRANSIENT_FAILURE or IDLE, re-resolve.
   // Only do this if we've started watching, not at startup time.
   // Otherwise, if the subchannel was already in state TRANSIENT_FAILURE
   // when the subchannel list was created, we'd wind up in a constant
   // loop of re-resolution.
   // Also attempt to reconnect.
-  if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+  if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE ||
+      connectivity_state == GRPC_CHANNEL_IDLE) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
       gpr_log(GPR_INFO,
-              "[RR %p] Subchannel %p has gone into TRANSIENT_FAILURE. "
-              "Requesting re-resolution",
-              p, subchannel());
+              "[RR %p] Subchannel %p reported %s; requesting re-resolution", p,
+              subchannel(), ConnectivityStateName(connectivity_state));
     }
     p->channel_control_helper()->RequestReresolution();
-    subchannel()->AttemptToConnect();
+    subchannel()->RequestConnection();
   }
-  // Update state counters.
-  UpdateConnectivityStateLocked(connectivity_state);
-  // Update overall state and renew notification.
-  subchannel_list()->UpdateRoundRobinStateFromSubchannelStateCountsLocked();
-}
-
-void RoundRobin::UpdateLocked(UpdateArgs args) {
-  ServerAddressList addresses;
-  if (args.addresses.ok()) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
-      gpr_log(GPR_INFO, "[RR %p] received update with %" PRIuPTR " addresses",
-              this, args.addresses->size());
-    }
-    addresses = std::move(*args.addresses);
-  } else {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
-      gpr_log(GPR_INFO, "[RR %p] received update with address error: %s", this,
-              args.addresses.status().ToString().c_str());
-    }
-    // If we already have a subchannel list, then ignore the resolver
-    // failure and keep using the existing list.
-    if (subchannel_list_ != nullptr) return;
-  }
-  // Replace latest_pending_subchannel_list_.
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace) &&
-      latest_pending_subchannel_list_ != nullptr) {
-    gpr_log(GPR_INFO,
-            "[RR %p] Shutting down previous pending subchannel list %p", this,
-            latest_pending_subchannel_list_.get());
-  }
-  latest_pending_subchannel_list_ = MakeOrphanable<RoundRobinSubchannelList>(
-      this, &grpc_lb_round_robin_trace, std::move(addresses), *args.args);
-  if (latest_pending_subchannel_list_->num_subchannels() == 0) {
-    // If the new list is empty, immediately promote the new list to the
-    // current list and transition to TRANSIENT_FAILURE.
-    absl::Status status =
-        args.addresses.ok() ? absl::UnavailableError(absl::StrCat(
-                                  "empty address list: ", args.resolution_note))
-                            : args.addresses.status();
-    channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_TRANSIENT_FAILURE, status,
-        absl::make_unique<TransientFailurePicker>(status));
-    subchannel_list_ = std::move(latest_pending_subchannel_list_);
-  } else if (subchannel_list_ == nullptr) {
-    // If there is no current list, immediately promote the new list to
-    // the current list and start watching it.
-    subchannel_list_ = std::move(latest_pending_subchannel_list_);
-    subchannel_list_->StartWatchingLocked();
-  } else {
-    // Start watching the pending list.  It will get swapped into the
-    // current list when it reports READY.
-    latest_pending_subchannel_list_->StartWatchingLocked();
+  // Update logical connectivity state.
+  // If it changed, update the policy state.
+  if (UpdateLogicalConnectivityStateLocked(connectivity_state)) {
+    subchannel_list()->MaybeUpdateRoundRobinConnectivityStateLocked(
+        absl::UnavailableError("connections to all backends failing"));
   }
 }
 
+//
+// factory
+//
+
 class RoundRobinConfig : public LoadBalancingPolicy::Config {
  public:
   const char* name() const override { return kRoundRobin; }
 };
 
-//
-// factory
-//
-
 class RoundRobinFactory : public LoadBalancingPolicyFactory {
  public:
   OrphanablePtr<LoadBalancingPolicy> CreateLoadBalancingPolicy(