grpc 1.53.0 → 1.55.0

data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc

@@ -22,7 +22,6 @@
 #include <stdlib.h>
 
 #include <algorithm>
-#include <atomic>
 #include <cmath>
 #include <memory>
 #include <string>
@@ -30,7 +29,6 @@
 #include <vector>
 
 #include "absl/base/attributes.h"
-#include "absl/base/thread_annotations.h"
 #include "absl/container/inlined_vector.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
@@ -39,15 +37,14 @@
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 
-#include <grpc/grpc.h>
-
 #define XXH_INLINE_ALL
 #include "xxhash.h"
 
+#include <grpc/grpc.h>
 #include <grpc/impl/connectivity_state.h>
 #include <grpc/support/log.h>
 
-#include "src/core/ext/filters/client_channel/client_channel.h"
+#include "src/core/ext/filters/client_channel/client_channel_internal.h"
 #include "src/core/ext/filters/client_channel/lb_policy/subchannel_list.h"
 #include "src/core/lib/address_utils/sockaddr_utils.h"
 #include "src/core/lib/channel/channel_args.h"
@@ -55,8 +52,8 @@
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
-#include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/gprpp/unique_type_name.h"
 #include "src/core/lib/gprpp/work_serializer.h"
 #include "src/core/lib/iomgr/closure.h"
@@ -143,8 +140,6 @@ class RingHash : public LoadBalancingPolicy {
   void ResetBackoffLocked() override;
 
  private:
-  ~RingHash() override;
-
   // Forward declaration.
   class RingHashSubchannelList;
 
@@ -165,13 +160,11 @@ class RingHash : public LoadBalancingPolicy {
 
     const ServerAddress& address() const { return address_; }
 
-    grpc_connectivity_state GetConnectivityState() const {
-      return connectivity_state_.load(std::memory_order_relaxed);
+    grpc_connectivity_state logical_connectivity_state() const {
+      return logical_connectivity_state_;
     }
-
-    absl::Status GetConnectivityStatus() const {
-      MutexLock lock(&mu_);
-      return connectivity_status_;
+    const absl::Status& logical_connectivity_status() const {
+      return logical_connectivity_status_;
     }
 
    private:
@@ -188,20 +181,28 @@ class RingHash : public LoadBalancingPolicy {
     // subchannel in some cases; for example, once this is set to
     // TRANSIENT_FAILURE, we do not change it again until we get READY,
     // so we skip any interim stops in CONNECTING.
-    // Uses an atomic so that it can be accessed outside of the WorkSerializer.
-    std::atomic<grpc_connectivity_state> connectivity_state_{GRPC_CHANNEL_IDLE};
-
-    mutable Mutex mu_;
-    absl::Status connectivity_status_ ABSL_GUARDED_BY(&mu_);
+    grpc_connectivity_state logical_connectivity_state_ = GRPC_CHANNEL_IDLE;
+    absl::Status logical_connectivity_status_;
   };
 
   // A list of subchannels and the ring containing those subchannels.
   class RingHashSubchannelList
       : public SubchannelList<RingHashSubchannelList, RingHashSubchannelData> {
    public:
-    struct RingEntry {
-      uint64_t hash;
-      RingHashSubchannelData* subchannel;
+    class Ring : public RefCounted<Ring> {
+     public:
+      struct RingEntry {
+        uint64_t hash;
+        size_t subchannel_index;
+      };
+
+      Ring(RingHashLbConfig* config, RingHashSubchannelList* subchannel_list,
+           const ChannelArgs& args);
+
+      const std::vector<RingEntry>& ring() const { return ring_; }
+
+     private:
+      std::vector<RingEntry> ring_;
     };
 
     RingHashSubchannelList(RingHash* policy, ServerAddressList addresses,
@@ -212,7 +213,7 @@ class RingHash : public LoadBalancingPolicy {
       p->Unref(DEBUG_LOCATION, "subchannel_list");
     }
 
-    const std::vector<RingEntry>& ring() const { return ring_; }
+    RefCountedPtr<Ring> ring() { return ring_; }
 
     // Updates the counters of subchannels in each state when a
     // subchannel transitions from old_state to new_state.
@@ -236,7 +237,7 @@ class RingHash : public LoadBalancingPolicy {
     size_t num_connecting_ = 0;
     size_t num_transient_failure_ = 0;
 
-    std::vector<RingEntry> ring_;
+    RefCountedPtr<Ring> ring_;
 
     // The index of the subchannel currently doing an internally
     // triggered connection attempt, if any.
@@ -251,24 +252,31 @@ class RingHash : public LoadBalancingPolicy {
 
   class Picker : public SubchannelPicker {
    public:
-    explicit Picker(RefCountedPtr<RingHashSubchannelList> subchannel_list)
-        : subchannel_list_(std::move(subchannel_list)) {}
-
-    ~Picker() override {
-      // Hop into WorkSerializer to unref the subchannel list, since that may
-      // trigger the unreffing of the underlying subchannels.
-      MakeOrphanable<WorkSerializerRunner>(std::move(subchannel_list_));
+    Picker(RefCountedPtr<RingHash> ring_hash_lb,
+           RingHashSubchannelList* subchannel_list)
+        : ring_hash_lb_(std::move(ring_hash_lb)),
+          ring_(subchannel_list->ring()) {
+      subchannels_.reserve(subchannel_list->num_subchannels());
+      for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) {
+        RingHashSubchannelData* subchannel_data =
+            subchannel_list->subchannel(i);
+        subchannels_.emplace_back(
+            SubchannelInfo{subchannel_data->subchannel()->Ref(),
+                           subchannel_data->logical_connectivity_state(),
+                           subchannel_data->logical_connectivity_status()});
+      }
     }
 
     PickResult Pick(PickArgs args) override;
 
    private:
-    // An interface for running a callback in the control plane WorkSerializer.
-    class WorkSerializerRunner : public Orphanable {
+    // A fire-and-forget class that schedules subchannel connection attempts
+    // on the control plane WorkSerializer.
+    class SubchannelConnectionAttempter : public Orphanable {
      public:
-      explicit WorkSerializerRunner(
-          RefCountedPtr<RingHashSubchannelList> subchannel_list)
-          : subchannel_list_(std::move(subchannel_list)) {
+      explicit SubchannelConnectionAttempter(
+          RefCountedPtr<RingHash> ring_hash_lb)
+          : ring_hash_lb_(std::move(ring_hash_lb)) {
         GRPC_CLOSURE_INIT(&closure_, RunInExecCtx, this, nullptr);
       }
 
@@ -278,56 +286,43 @@ class RingHash : public LoadBalancingPolicy {
         ExecCtx::Run(DEBUG_LOCATION, &closure_, absl::OkStatus());
       }
 
-      // Will be invoked inside of the WorkSerializer.
-      virtual void Run() {}
-
-     protected:
-      RingHash* ring_hash_lb() const {
-        return static_cast<RingHash*>(subchannel_list_->policy());
+      void AddSubchannel(RefCountedPtr<SubchannelInterface> subchannel) {
+        subchannels_.push_back(std::move(subchannel));
       }
 
      private:
       static void RunInExecCtx(void* arg, grpc_error_handle /*error*/) {
-        auto* self = static_cast<WorkSerializerRunner*>(arg);
-        self->ring_hash_lb()->work_serializer()->Run(
+        auto* self = static_cast<SubchannelConnectionAttempter*>(arg);
+        self->ring_hash_lb_->work_serializer()->Run(
             [self]() {
-              self->Run();
+              if (!self->ring_hash_lb_->shutdown_) {
+                for (auto& subchannel : self->subchannels_) {
+                  subchannel->RequestConnection();
+                }
+              }
               delete self;
             },
             DEBUG_LOCATION);
       }
 
-      RefCountedPtr<RingHashSubchannelList> subchannel_list_;
+      RefCountedPtr<RingHash> ring_hash_lb_;
       grpc_closure closure_;
+      std::vector<RefCountedPtr<SubchannelInterface>> subchannels_;
     };
 
-    // A fire-and-forget class that schedules subchannel connection attempts
-    // on the control plane WorkSerializer.
-    class SubchannelConnectionAttempter : public WorkSerializerRunner {
-     public:
-      explicit SubchannelConnectionAttempter(
-          RefCountedPtr<RingHashSubchannelList> subchannel_list)
-          : WorkSerializerRunner(std::move(subchannel_list)) {}
-
-      void AddSubchannel(RefCountedPtr<SubchannelInterface> subchannel) {
-        subchannels_.push_back(std::move(subchannel));
-      }
-
-      void Run() override {
-        if (!ring_hash_lb()->shutdown_) {
-          for (auto& subchannel : subchannels_) {
-            subchannel->RequestConnection();
-          }
-        }
-      }
-
-     private:
-      std::vector<RefCountedPtr<SubchannelInterface>> subchannels_;
+    struct SubchannelInfo {
+      RefCountedPtr<SubchannelInterface> subchannel;
+      grpc_connectivity_state state;
+      absl::Status status;
     };
 
-    RefCountedPtr<RingHashSubchannelList> subchannel_list_;
+    RefCountedPtr<RingHash> ring_hash_lb_;
+    RefCountedPtr<RingHashSubchannelList::Ring> ring_;
+    std::vector<SubchannelInfo> subchannels_;
   };
 
+  ~RingHash() override;
+
   void ShutdownLocked() override;
 
   // Current config from resolver.
@@ -345,15 +340,14 @@ class RingHash : public LoadBalancingPolicy {
 //
 
 RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
-  auto* call_state = static_cast<ClientChannel::LoadBalancedCall::LbCallState*>(
-      args.call_state);
+  auto* call_state = static_cast<ClientChannelLbCallState*>(args.call_state);
   auto hash = call_state->GetCallAttribute(RequestHashAttributeName());
   uint64_t h;
   if (!absl::SimpleAtoi(hash, &h)) {
     return PickResult::Fail(
         absl::InternalError("ring hash value is not a number"));
   }
-  const auto& ring = subchannel_list_->ring();
+  const auto& ring = ring_->ring();
   // Ported from https://github.com/RJ/ketama/blob/master/libketama/ketama.c
   // (ketama_get_server) NOTE: The algorithm depends on using signed integers
   // for lowp, highp, and first_index. Do not change them!
@@ -386,27 +380,25 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
       [&](RefCountedPtr<SubchannelInterface> subchannel) {
         if (subchannel_connection_attempter == nullptr) {
           subchannel_connection_attempter =
-              MakeOrphanable<SubchannelConnectionAttempter>(
-                  subchannel_list_->Ref(DEBUG_LOCATION,
-                                        "SubchannelConnectionAttempter"));
+              MakeOrphanable<SubchannelConnectionAttempter>(ring_hash_lb_->Ref(
+                  DEBUG_LOCATION, "SubchannelConnectionAttempter"));
         }
         subchannel_connection_attempter->AddSubchannel(std::move(subchannel));
       };
-  switch (ring[first_index].subchannel->GetConnectivityState()) {
+  SubchannelInfo& first_subchannel =
+      subchannels_[ring[first_index].subchannel_index];
+  switch (first_subchannel.state) {
     case GRPC_CHANNEL_READY:
-      return PickResult::Complete(
-          ring[first_index].subchannel->subchannel()->Ref());
+      return PickResult::Complete(first_subchannel.subchannel);
     case GRPC_CHANNEL_IDLE:
-      ScheduleSubchannelConnectionAttempt(
-          ring[first_index].subchannel->subchannel()->Ref());
+      ScheduleSubchannelConnectionAttempt(first_subchannel.subchannel);
       ABSL_FALLTHROUGH_INTENDED;
     case GRPC_CHANNEL_CONNECTING:
       return PickResult::Queue();
     default:  // GRPC_CHANNEL_TRANSIENT_FAILURE
       break;
   }
-  ScheduleSubchannelConnectionAttempt(
-      ring[first_index].subchannel->subchannel()->Ref());
+  ScheduleSubchannelConnectionAttempt(first_subchannel.subchannel);
   // Loop through remaining subchannels to find one in READY.
   // On the way, we make sure the right set of connection attempts
   // will happen.
@@ -414,19 +406,17 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
   bool found_first_non_failed = false;
   for (size_t i = 1; i < ring.size(); ++i) {
     const auto& entry = ring[(first_index + i) % ring.size()];
-    if (entry.subchannel == ring[first_index].subchannel) {
+    if (entry.subchannel_index == ring[first_index].subchannel_index) {
       continue;
     }
-    grpc_connectivity_state connectivity_state =
-        entry.subchannel->GetConnectivityState();
-    if (connectivity_state == GRPC_CHANNEL_READY) {
-      return PickResult::Complete(entry.subchannel->subchannel()->Ref());
+    SubchannelInfo& subchannel_info = subchannels_[entry.subchannel_index];
+    if (subchannel_info.state == GRPC_CHANNEL_READY) {
+      return PickResult::Complete(subchannel_info.subchannel);
     }
     if (!found_second_subchannel) {
-      switch (connectivity_state) {
+      switch (subchannel_info.state) {
         case GRPC_CHANNEL_IDLE:
-          ScheduleSubchannelConnectionAttempt(
-              entry.subchannel->subchannel()->Ref());
+          ScheduleSubchannelConnectionAttempt(subchannel_info.subchannel);
           ABSL_FALLTHROUGH_INTENDED;
         case GRPC_CHANNEL_CONNECTING:
           return PickResult::Queue();
@@ -436,13 +426,11 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
       found_second_subchannel = true;
     }
     if (!found_first_non_failed) {
-      if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-        ScheduleSubchannelConnectionAttempt(
-            entry.subchannel->subchannel()->Ref());
+      if (subchannel_info.state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+        ScheduleSubchannelConnectionAttempt(subchannel_info.subchannel);
       } else {
-        if (connectivity_state == GRPC_CHANNEL_IDLE) {
-          ScheduleSubchannelConnectionAttempt(
-              entry.subchannel->subchannel()->Ref());
+        if (subchannel_info.state == GRPC_CHANNEL_IDLE) {
+          ScheduleSubchannelConnectionAttempt(subchannel_info.subchannel);
         }
         found_first_non_failed = true;
       }
@@ -450,27 +438,16 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
   }
   return PickResult::Fail(absl::UnavailableError(absl::StrCat(
       "ring hash cannot find a connected subchannel; first failure: ",
-      ring[first_index].subchannel->GetConnectivityStatus().ToString())));
+      first_subchannel.status.ToString())));
 }
 
 //
-// RingHash::RingHashSubchannelList
+// RingHash::RingHashSubchannelList::Ring
 //
 
-RingHash::RingHashSubchannelList::RingHashSubchannelList(
-    RingHash* policy, ServerAddressList addresses, const ChannelArgs& args)
-    : SubchannelList(policy,
-                     (GRPC_TRACE_FLAG_ENABLED(grpc_lb_ring_hash_trace)
-                          ? "RingHashSubchannelList"
-                          : nullptr),
-                     std::move(addresses), policy->channel_control_helper(),
-                     args),
-      num_idle_(num_subchannels()) {
-  // Need to maintain a ref to the LB policy as long as we maintain
-  // any references to subchannels, since the subchannels'
-  // pollset_sets will include the LB policy's pollset_set.
-  policy->Ref(DEBUG_LOCATION, "subchannel_list").release();
-  // Construct the ring.
+RingHash::RingHashSubchannelList::Ring::Ring(
+    RingHashLbConfig* config, RingHashSubchannelList* subchannel_list,
+    const ChannelArgs& args) {
   // Store the weights while finding the sum.
   struct AddressWeight {
     std::string address;
@@ -481,9 +458,9 @@ RingHash::RingHashSubchannelList::RingHashSubchannelList(
   };
   std::vector<AddressWeight> address_weights;
   size_t sum = 0;
-  address_weights.reserve(num_subchannels());
-  for (size_t i = 0; i < num_subchannels(); ++i) {
-    RingHashSubchannelData* sd = subchannel(i);
+  address_weights.reserve(subchannel_list->num_subchannels());
+  for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) {
+    RingHashSubchannelData* sd = subchannel_list->subchannel(i);
     const ServerAddressWeightAttribute* weight_attribute = static_cast<
         const ServerAddressWeightAttribute*>(sd->address().GetAttribute(
         ServerAddressWeightAttribute::kServerAddressWeightAttributeKey));
@@ -517,10 +494,8 @@ RingHash::RingHashSubchannelList::RingHashSubchannelList(
   // to fit.
   const size_t ring_size_cap = args.GetInt(GRPC_ARG_RING_HASH_LB_RING_SIZE_CAP)
                                    .value_or(kRingSizeCapDefault);
-  const size_t min_ring_size =
-      std::min(policy->config_->min_ring_size(), ring_size_cap);
-  const size_t max_ring_size =
-      std::min(policy->config_->max_ring_size(), ring_size_cap);
+  const size_t min_ring_size = std::min(config->min_ring_size(), ring_size_cap);
+  const size_t max_ring_size = std::min(config->max_ring_size(), ring_size_cap);
   const double scale = std::min(
       std::ceil(min_normalized_weight * min_ring_size) / min_normalized_weight,
       static_cast<double>(max_ring_size));
@@ -537,7 +512,7 @@ RingHash::RingHashSubchannelList::RingHashSubchannelList(
   double target_hashes = 0.0;
   uint64_t min_hashes_per_host = ring_size;
   uint64_t max_hashes_per_host = 0;
-  for (size_t i = 0; i < num_subchannels(); ++i) {
+  for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) {
     const std::string& address_string = address_weights[i].address;
     hash_key_buffer.assign(address_string.begin(), address_string.end());
     hash_key_buffer.emplace_back('_');
@@ -550,7 +525,7 @@ RingHash::RingHashSubchannelList::RingHashSubchannelList(
       absl::string_view hash_key(hash_key_buffer.data(),
                                  hash_key_buffer.size());
       const uint64_t hash = XXH64(hash_key.data(), hash_key.size(), 0);
-      ring_.push_back({hash, subchannel(i)});
+      ring_.push_back({hash, i});
       ++count;
       ++current_hashes;
       hash_key_buffer.erase(offset_start, hash_key_buffer.end());
@@ -561,14 +536,34 @@ RingHash::RingHashSubchannelList::RingHashSubchannelList(
         std::max(static_cast<uint64_t>(i), max_hashes_per_host);
   }
   std::sort(ring_.begin(), ring_.end(),
-            [](const RingHashSubchannelList::RingEntry& lhs,
-               const RingHashSubchannelList::RingEntry& rhs) -> bool {
+            [](const RingEntry& lhs, const RingEntry& rhs) -> bool {
               return lhs.hash < rhs.hash;
             });
+}
+
+//
+// RingHash::RingHashSubchannelList
+//
+
+RingHash::RingHashSubchannelList::RingHashSubchannelList(
+    RingHash* policy, ServerAddressList addresses, const ChannelArgs& args)
+    : SubchannelList(policy,
+                     (GRPC_TRACE_FLAG_ENABLED(grpc_lb_ring_hash_trace)
+                          ? "RingHashSubchannelList"
+                          : nullptr),
+                     std::move(addresses), policy->channel_control_helper(),
+                     args),
+      num_idle_(num_subchannels()) {
+  // Need to maintain a ref to the LB policy as long as we maintain
+  // any references to subchannels, since the subchannels'
+  // pollset_sets will include the LB policy's pollset_set.
+  policy->Ref(DEBUG_LOCATION, "subchannel_list").release();
+  // Construct the ring.
+  ring_ = MakeRefCounted<Ring>(policy->config_.get(), this, args);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_ring_hash_trace)) {
     gpr_log(GPR_INFO,
             "[RH %p] created subchannel list %p with %" PRIuPTR " ring entries",
-            policy, this, ring_.size());
+            policy, this, ring_->ring().size());
   }
 }
 
@@ -660,7 +655,7 @@ void RingHash::RingHashSubchannelList::UpdateRingHashConnectivityStateLocked(
   // Note that we use our own picker regardless of connectivity state.
   p->channel_control_helper()->UpdateState(
       state, status,
-      MakeRefCounted<Picker>(Ref(DEBUG_LOCATION, "RingHashPicker")));
+      MakeRefCounted<Picker>(p->Ref(DEBUG_LOCATION, "RingHashPicker"), this));
   // While the ring_hash policy is reporting TRANSIENT_FAILURE, it will
   // not be getting any pick requests from the priority policy.
   // However, because the ring_hash policy does not attempt to
@@ -707,7 +702,6 @@ void RingHash::RingHashSubchannelData::ProcessConnectivityChangeLocked(
     absl::optional<grpc_connectivity_state> old_state,
     grpc_connectivity_state new_state) {
   RingHash* p = static_cast<RingHash*>(subchannel_list()->policy());
-  grpc_connectivity_state last_connectivity_state = GetConnectivityState();
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_ring_hash_trace)) {
     gpr_log(
         GPR_INFO,
@@ -715,7 +709,7 @@ void RingHash::RingHashSubchannelData::ProcessConnectivityChangeLocked(
         "(index %" PRIuPTR " of %" PRIuPTR "): prev_state=%s new_state=%s",
         p, subchannel(), subchannel_list(), Index(),
         subchannel_list()->num_subchannels(),
-        ConnectivityStateName(last_connectivity_state),
+        ConnectivityStateName(logical_connectivity_state_),
         ConnectivityStateName(new_state));
   }
   GPR_ASSERT(subchannel() != nullptr);
@@ -735,34 +729,23 @@ void RingHash::RingHashSubchannelData::ProcessConnectivityChangeLocked(
   const bool connection_attempt_complete = new_state != GRPC_CHANNEL_CONNECTING;
   // Decide what state to report for the purposes of aggregation and
   // picker behavior.
-  // If the last recorded state was TRANSIENT_FAILURE, ignore the update
-  // unless the new state is READY.
-  bool update_status = true;
-  absl::Status status = connectivity_status();
-  if (last_connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE &&
-      new_state != GRPC_CHANNEL_READY &&
-      new_state != GRPC_CHANNEL_TRANSIENT_FAILURE) {
-    new_state = GRPC_CHANNEL_TRANSIENT_FAILURE;
-    {
-      MutexLock lock(&mu_);
-      status = connectivity_status_;
-    }
-    update_status = false;
-  }
-  // Update state counters used for aggregation.
-  subchannel_list()->UpdateStateCountersLocked(last_connectivity_state,
-                                               new_state);
-  // Update status seen by picker if needed.
-  if (update_status) {
-    MutexLock lock(&mu_);
-    connectivity_status_ = connectivity_status();
+  // If the last recorded state was TRANSIENT_FAILURE, ignore the change
+  // unless the new state is READY (or TF again, in which case we need
+  // to update the status).
+  if (logical_connectivity_state_ != GRPC_CHANNEL_TRANSIENT_FAILURE ||
+      new_state == GRPC_CHANNEL_READY ||
+      new_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+    // Update state counters used for aggregation.
+    subchannel_list()->UpdateStateCountersLocked(logical_connectivity_state_,
+                                                 new_state);
+    // Update logical state.
+    logical_connectivity_state_ = new_state;
+    logical_connectivity_status_ = connectivity_status();
   }
-  // Update last seen state, also used by picker.
-  connectivity_state_.store(new_state, std::memory_order_relaxed);
   // Update the RH policy's connectivity state, creating new picker and new
   // ring.
   subchannel_list()->UpdateRingHashConnectivityStateLocked(
-      Index(), connection_attempt_complete, status);
+      Index(), connection_attempt_complete, logical_connectivity_status_);
 }
 
 //