grpc 1.41.1 → 1.44.0.pre2

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc

@@ -1,20 +1,18 @@
-/*
- *
- * Copyright 2016 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
+//
+// Copyright 2016 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 
 /// Implementation of the gRPC LB policy.
 ///
@@ -82,12 +80,13 @@
 #include "src/core/ext/filters/client_channel/lb_policy_factory.h"
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
-#include "src/core/ext/filters/client_channel/server_address.h"
 #include "src/core/lib/address_utils/parse_address.h"
 #include "src/core/lib/address_utils/sockaddr_utils.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/gprpp/memory.h"
@@ -96,29 +95,23 @@
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/socket_utils.h"
 #include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/resolver/server_address.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
-#include "src/core/lib/surface/channel_init.h"
-#include "src/core/lib/transport/static_metadata.h"
 
 #define GRPC_GRPCLB_INITIAL_CONNECT_BACKOFF_SECONDS 1
 #define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6
 #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120
 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2
 #define GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS 10000
-
-#define GRPC_ARG_GRPCLB_ADDRESS_LB_TOKEN "grpc.grpclb_address_lb_token"
-#define GRPC_ARG_GRPCLB_ADDRESS_CLIENT_STATS "grpc.grpclb_address_client_stats"
+#define GRPC_GRPCLB_DEFAULT_SUBCHANNEL_DELETION_DELAY_MS 10000
 
 namespace grpc_core {
 
 TraceFlag grpc_lb_glb_trace(false, "glb");
 
-const char kGrpcLbClientStatsMetadataKey[] = "grpclb_client_stats";
-const char kGrpcLbLbTokenMetadataKey[] = "lb-token";
-
 const char kGrpcLbAddressAttributeKey[] = "grpclb";
 
 namespace {
@@ -233,16 +226,24 @@ class GrpcLb : public LoadBalancingPolicy {
   class SubchannelWrapper : public DelegatingSubchannel {
    public:
     SubchannelWrapper(RefCountedPtr<SubchannelInterface> subchannel,
-                      std::string lb_token,
+                      RefCountedPtr<GrpcLb> lb_policy, std::string lb_token,
                       RefCountedPtr<GrpcLbClientStats> client_stats)
         : DelegatingSubchannel(std::move(subchannel)),
+          lb_policy_(std::move(lb_policy)),
           lb_token_(std::move(lb_token)),
           client_stats_(std::move(client_stats)) {}
 
+    ~SubchannelWrapper() override {
+      if (!lb_policy_->shutting_down_) {
+        lb_policy_->CacheDeletedSubchannelLocked(wrapped_subchannel());
+      }
+    }
+
     const std::string& lb_token() const { return lb_token_; }
     GrpcLbClientStats* client_stats() const { return client_stats_.get(); }
 
    private:
+    RefCountedPtr<GrpcLb> lb_policy_;
     std::string lb_token_;
     RefCountedPtr<GrpcLbClientStats> client_stats_;
   };
@@ -265,7 +266,7 @@ class GrpcLb : public LoadBalancingPolicy {
           static_cast<const TokenAndClientStatsAttribute*>(other_base);
       int r = lb_token_.compare(other->lb_token_);
       if (r != 0) return r;
-      return GPR_ICMP(client_stats_.get(), other->client_stats_.get());
+      return QsortCompare(client_stats_.get(), other->client_stats_.get());
     }
 
     std::string ToString() const override {
@@ -351,6 +352,7 @@ class GrpcLb : public LoadBalancingPolicy {
     void UpdateState(grpc_connectivity_state state, const absl::Status& status,
                      std::unique_ptr<SubchannelPicker> picker) override;
     void RequestReresolution() override;
+    absl::string_view GetAuthority() override;
     void AddTraceEvent(TraceSeverity severity,
                        absl::string_view message) override;
 
@@ -395,10 +397,7 @@ class GrpcLb : public LoadBalancingPolicy {
   void ShutdownLocked() override;
 
   // Helper functions used in UpdateLocked().
-  void ProcessAddressesAndChannelArgsLocked(const ServerAddressList& addresses,
-                                            const grpc_channel_args& args);
-  static ServerAddressList AddNullLbTokenToAddresses(
-      const ServerAddressList& addresses);
+  void UpdateBalancerChannelLocked(const grpc_channel_args& args);
 
   void CancelBalancerChannelConnectivityWatchLocked();
 
@@ -420,6 +419,13 @@ class GrpcLb : public LoadBalancingPolicy {
       const grpc_channel_args* args);
   void CreateOrUpdateChildPolicyLocked();
 
+  // Subchannel caching.
+  void CacheDeletedSubchannelLocked(
+      RefCountedPtr<SubchannelInterface> subchannel);
+  void StartSubchannelCacheTimerLocked();
+  static void OnSubchannelCacheTimer(void* arg, grpc_error_handle error);
+  void OnSubchannelCacheTimerLocked(grpc_error_handle error);
+
   // Who the client is trying to communicate with.
   std::string server_name_;
   // Configurations for the policy.
@@ -446,7 +452,7 @@ class GrpcLb : public LoadBalancingPolicy {
   // contains a non-NULL lb_call_.
   OrphanablePtr<BalancerCallState> lb_calld_;
   // Timeout in milliseconds for the LB call. 0 means no deadline.
-  int lb_call_timeout_ms_ = 0;
+  const int lb_call_timeout_ms_ = 0;
   // Balancer call retry state.
   BackOff lb_call_backoff_;
   bool retry_timer_callback_pending_ = false;
@@ -460,11 +466,14 @@ class GrpcLb : public LoadBalancingPolicy {
   // Whether we're in fallback mode.
   bool fallback_mode_ = false;
   // The backend addresses from the resolver.
-  ServerAddressList fallback_backend_addresses_;
+  absl::StatusOr<ServerAddressList> fallback_backend_addresses_;
+  // The last resolution note from our parent.
+  // To be passed to child policy when fallback_backend_addresses_ is empty.
+  std::string resolution_note_;
   // State for fallback-at-startup checks.
   // Timeout after startup after which we will go into fallback mode if
   // we have not received a serverlist from the balancer.
-  int fallback_at_startup_timeout_ = 0;
+  const int fallback_at_startup_timeout_ = 0;
   bool fallback_at_startup_checks_pending_ = false;
   grpc_timer lb_fallback_timer_;
   grpc_closure lb_on_fallback_;
@@ -473,6 +482,15 @@ class GrpcLb : public LoadBalancingPolicy {
   OrphanablePtr<LoadBalancingPolicy> child_policy_;
   // Child policy in state READY.
   bool child_policy_ready_ = false;
+
+  // Deleted subchannel caching.
+  const grpc_millis subchannel_cache_interval_ms_;
+  std::map<grpc_millis /*deletion time*/,
+           std::vector<RefCountedPtr<SubchannelInterface>>>
+      cached_subchannels_;
+  grpc_timer subchannel_cache_timer_;
+  grpc_closure on_subchannel_cache_timer_;
+  bool subchannel_cache_timer_pending_ = false;
 };
 
 //
@@ -631,7 +649,7 @@ GrpcLb::PickResult GrpcLb::Picker::Pick(PickArgs args) {
       // a string and rely on the client_load_reporting filter to know
       // how to interpret it.
       args.initial_metadata->Add(
-          kGrpcLbClientStatsMetadataKey,
+          GrpcLbClientStatsMetadata::key(),
           absl::string_view(reinterpret_cast<const char*>(client_stats), 0));
       // Update calls-started.
       client_stats->AddCallStarted();
@@ -644,7 +662,7 @@ GrpcLb::PickResult GrpcLb::Picker::Pick(PickArgs args) {
       char* lb_token = static_cast<char*>(
           args.call_state->Alloc(subchannel_wrapper->lb_token().size() + 1));
       strcpy(lb_token, subchannel_wrapper->lb_token().c_str());
-      args.initial_metadata->Add(kGrpcLbLbTokenMetadataKey, lb_token);
+      args.initial_metadata->Add(LbTokenMetadata::key(), lb_token);
     }
     // Unwrap subchannel to pass up to the channel.
     complete_pick->subchannel = subchannel_wrapper->wrapped_subchannel();
@@ -673,7 +691,8 @@ RefCountedPtr<SubchannelInterface> GrpcLb::Helper::CreateSubchannel(
   return MakeRefCounted<SubchannelWrapper>(
       parent_->channel_control_helper()->CreateSubchannel(std::move(address),
                                                           args),
-      std::move(lb_token), std::move(client_stats));
+      parent_->Ref(DEBUG_LOCATION, "SubchannelWrapper"), std::move(lb_token),
+      std::move(client_stats));
 }
 
 void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
@@ -729,6 +748,10 @@ void GrpcLb::Helper::RequestReresolution() {
   }
 }
 
+absl::string_view GrpcLb::Helper::GetAuthority() {
+  return parent_->channel_control_helper()->GetAuthority();
+}
+
 void GrpcLb::Helper::AddTraceEvent(TraceSeverity severity,
                                    absl::string_view message) {
   if (parent_->shutting_down_) return;
@@ -767,7 +790,7 @@ GrpcLb::BalancerCallState::BalancerCallState(
   lb_call_ = grpc_channel_create_pollset_set_call(
       grpclb_policy()->lb_channel_, nullptr, GRPC_PROPAGATE_DEFAULTS,
       grpclb_policy_->interested_parties(),
-      GRPC_MDSTR_SLASH_GRPC_DOT_LB_DOT_V1_DOT_LOADBALANCER_SLASH_BALANCELOAD,
+      Slice::FromStaticString("/grpc.lb.v1.LoadBalancer/BalanceLoad").c_slice(),
       nullptr, deadline, nullptr);
   // Init the LB call request payload.
   upb::Arena arena;
@@ -902,7 +925,7 @@ void GrpcLb::BalancerCallState::ScheduleNextClientLoadReportLocked() {
 void GrpcLb::BalancerCallState::MaybeSendClientLoadReport(
     void* arg, grpc_error_handle error) {
   BalancerCallState* lb_calld = static_cast<BalancerCallState*>(arg);
-  GRPC_ERROR_REF(error);  // ref owned by lambda
+  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
   lb_calld->grpclb_policy()->work_serializer()->Run(
       [lb_calld, error]() { lb_calld->MaybeSendClientLoadReportLocked(error); },
       DEBUG_LOCATION);
@@ -981,7 +1004,7 @@ void GrpcLb::BalancerCallState::SendClientLoadReportLocked() {
 void GrpcLb::BalancerCallState::ClientLoadReportDone(void* arg,
                                                      grpc_error_handle error) {
   BalancerCallState* lb_calld = static_cast<BalancerCallState*>(arg);
-  GRPC_ERROR_REF(error);  // ref owned by lambda
+  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
   lb_calld->grpclb_policy()->work_serializer()->Run(
       [lb_calld, error]() { lb_calld->ClientLoadReportDoneLocked(error); },
       DEBUG_LOCATION);
@@ -1054,8 +1077,8 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked() {
     switch (response.type) {
       case response.INITIAL: {
         if (response.client_stats_report_interval != 0) {
-          client_stats_report_interval_ =
-              GPR_MAX(GPR_MS_PER_SEC, response.client_stats_report_interval);
+          client_stats_report_interval_ = std::max(
+              int64_t(GPR_MS_PER_SEC), response.client_stats_report_interval);
           if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
             gpr_log(GPR_INFO,
                     "[grpclb %p] lb_calld=%p: Received initial LB response "
@@ -1184,7 +1207,7 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked() {
 void GrpcLb::BalancerCallState::OnBalancerStatusReceived(
     void* arg, grpc_error_handle error) {
   BalancerCallState* lb_calld = static_cast<BalancerCallState*>(arg);
-  GRPC_ERROR_REF(error);  // owned by lambda
+  (void)GRPC_ERROR_REF(error);  // owned by lambda
   lb_calld->grpclb_policy()->work_serializer()->Run(
       [lb_calld, error]() { lb_calld->OnBalancerStatusReceivedLocked(error); },
       DEBUG_LOCATION);
@@ -1295,8 +1318,7 @@ grpc_channel_args* BuildBalancerChannelArgs(
   absl::InlinedVector<grpc_arg, 3> args_to_add = {
       // The fake resolver response generator, which we use to inject
       // address updates into the LB channel.
-      grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
-          response_generator),
+      FakeResolverResponseGenerator::MakeChannelArg(response_generator),
       // A channel arg indicating the target is a grpclb load balancer.
       grpc_channel_arg_integer_create(
           const_cast<char*>(GRPC_ARG_ADDRESS_IS_GRPCLB_LOAD_BALANCER), 1),
@@ -1316,9 +1338,21 @@ grpc_channel_args* BuildBalancerChannelArgs(
 // ctor and dtor
 //
 
+std::string GetServerNameFromChannelArgs(const grpc_channel_args* args) {
+  const char* server_uri =
+      grpc_channel_args_find_string(args, GRPC_ARG_SERVER_URI);
+  GPR_ASSERT(server_uri != nullptr);
+  absl::StatusOr<URI> uri = URI::Parse(server_uri);
+  GPR_ASSERT(uri.ok() && !uri->path().empty());
+  return std::string(absl::StripPrefix(uri->path(), "/"));
+}
+
 GrpcLb::GrpcLb(Args args)
     : LoadBalancingPolicy(std::move(args)),
+      server_name_(GetServerNameFromChannelArgs(args.args)),
       response_generator_(MakeRefCounted<FakeResolverResponseGenerator>()),
+      lb_call_timeout_ms_(grpc_channel_args_find_integer(
+          args.args, GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS, {0, 0, INT_MAX})),
       lb_call_backoff_(
           BackOff::Options()
               .set_initial_backoff(GRPC_GRPCLB_INITIAL_CONNECT_BACKOFF_SECONDS *
@@ -1326,31 +1360,25 @@ GrpcLb::GrpcLb(Args args)
               .set_multiplier(GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER)
               .set_jitter(GRPC_GRPCLB_RECONNECT_JITTER)
               .set_max_backoff(GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS *
-                               1000)) {
-  // Closure Initialization
-  GRPC_CLOSURE_INIT(&lb_on_fallback_, &GrpcLb::OnFallbackTimer, this,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&lb_on_call_retry_, &GrpcLb::OnBalancerCallRetryTimer, this,
-                    grpc_schedule_on_exec_ctx);
-  // Record server name.
-  const grpc_arg* arg = grpc_channel_args_find(args.args, GRPC_ARG_SERVER_URI);
-  const char* server_uri = grpc_channel_arg_get_string(arg);
-  GPR_ASSERT(server_uri != nullptr);
-  absl::StatusOr<URI> uri = URI::Parse(server_uri);
-  GPR_ASSERT(uri.ok() && !uri->path().empty());
-  server_name_ = std::string(absl::StripPrefix(uri->path(), "/"));
+                               1000)),
+      fallback_at_startup_timeout_(grpc_channel_args_find_integer(
+          args.args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS,
+          {GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, INT_MAX})),
+      subchannel_cache_interval_ms_(grpc_channel_args_find_integer(
+          args.args, GRPC_ARG_GRPCLB_SUBCHANNEL_CACHE_INTERVAL_MS,
+          {GRPC_GRPCLB_DEFAULT_SUBCHANNEL_DELETION_DELAY_MS, 0, INT_MAX})) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO,
             "[grpclb %p] Will use '%s' as the server name for LB request.",
             this, server_name_.c_str());
   }
-  // Record LB call timeout.
-  arg = grpc_channel_args_find(args.args, GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS);
-  lb_call_timeout_ms_ = grpc_channel_arg_get_integer(arg, {0, 0, INT_MAX});
-  // Record fallback-at-startup timeout.
-  arg = grpc_channel_args_find(args.args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS);
-  fallback_at_startup_timeout_ = grpc_channel_arg_get_integer(
-      arg, {GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, INT_MAX});
+  // Closure Initialization
+  GRPC_CLOSURE_INIT(&lb_on_fallback_, &GrpcLb::OnFallbackTimer, this,
+                    grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_INIT(&lb_on_call_retry_, &GrpcLb::OnBalancerCallRetryTimer, this,
+                    grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_INIT(&on_subchannel_cache_timer_, &OnSubchannelCacheTimer, this,
+                    nullptr);
 }
 
 GrpcLb::~GrpcLb() { grpc_channel_args_destroy(args_); }
@@ -1358,6 +1386,11 @@ GrpcLb::~GrpcLb() { grpc_channel_args_destroy(args_); }
 void GrpcLb::ShutdownLocked() {
   shutting_down_ = true;
   lb_calld_.reset();
+  if (subchannel_cache_timer_pending_) {
+    subchannel_cache_timer_pending_ = false;
+    grpc_timer_cancel(&subchannel_cache_timer_);
+  }
+  cached_subchannels_.clear();
   if (retry_timer_callback_pending_) {
     grpc_timer_cancel(&lb_call_retry_timer_);
   }
@@ -1404,8 +1437,20 @@ void GrpcLb::UpdateLocked(UpdateArgs args) {
   const bool is_initial_update = lb_channel_ == nullptr;
   config_ = args.config;
   GPR_ASSERT(config_ != nullptr);
-  ProcessAddressesAndChannelArgsLocked(args.addresses, *args.args);
-  // Update the existing child policy.
+  // Update fallback address list.
+  fallback_backend_addresses_ = std::move(args.addresses);
+  if (fallback_backend_addresses_.ok()) {
+    // Add null LB token attributes.
+    for (ServerAddress& address : *fallback_backend_addresses_) {
+      address = address.WithAttribute(
+          kGrpcLbAddressAttributeKey,
+          absl::make_unique<TokenAndClientStatsAttribute>("", nullptr));
+    }
+  }
+  resolution_note_ = std::move(args.resolution_note);
+  // Update balancer channel.
+  UpdateBalancerChannelLocked(*args.args);
+  // Update the existing child policy, if any.
   if (child_policy_ != nullptr) CreateOrUpdateChildPolicyLocked();
   // If this is the initial update, start the fallback-at-startup checks
   // and the balancer call.
@@ -1434,21 +1479,7 @@ void GrpcLb::UpdateLocked(UpdateArgs args) {
 // helpers for UpdateLocked()
 //
 
-ServerAddressList GrpcLb::AddNullLbTokenToAddresses(
-    const ServerAddressList& addresses) {
-  ServerAddressList addresses_out;
-  for (const ServerAddress& address : addresses) {
-    addresses_out.emplace_back(address.WithAttribute(
-        kGrpcLbAddressAttributeKey,
-        absl::make_unique<TokenAndClientStatsAttribute>("", nullptr)));
-  }
-  return addresses_out;
-}
-
-void GrpcLb::ProcessAddressesAndChannelArgsLocked(
-    const ServerAddressList& addresses, const grpc_channel_args& args) {
-  // Update fallback address list.
-  fallback_backend_addresses_ = AddNullLbTokenToAddresses(addresses);
+void GrpcLb::UpdateBalancerChannelLocked(const grpc_channel_args& args) {
   // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args,
   // since we use this to trigger the client_load_reporting filter.
   static const char* args_to_remove[] = {GRPC_ARG_LB_POLICY_NAME};
@@ -1534,7 +1565,7 @@ void GrpcLb::StartBalancerCallRetryTimerLocked() {
 
 void GrpcLb::OnBalancerCallRetryTimer(void* arg, grpc_error_handle error) {
   GrpcLb* grpclb_policy = static_cast<GrpcLb*>(arg);
-  GRPC_ERROR_REF(error);  // ref owned by lambda
+  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
   grpclb_policy->work_serializer()->Run(
       [grpclb_policy, error]() {
         grpclb_policy->OnBalancerCallRetryTimerLocked(error);
@@ -1578,7 +1609,7 @@ void GrpcLb::MaybeEnterFallbackModeAfterStartup() {
 
 void GrpcLb::OnFallbackTimer(void* arg, grpc_error_handle error) {
   GrpcLb* grpclb_policy = static_cast<GrpcLb*>(arg);
-  GRPC_ERROR_REF(error);  // ref owned by lambda
+  (void)GRPC_ERROR_REF(error);  // ref owned by lambda
   grpclb_policy->work_serializer()->Run(
       [grpclb_policy, error]() { grpclb_policy->OnFallbackTimerLocked(error); },
       DEBUG_LOCATION);
@@ -1650,9 +1681,14 @@ void GrpcLb::CreateOrUpdateChildPolicyLocked() {
     // If CreateOrUpdateChildPolicyLocked() is invoked when we haven't
     // received any serverlist from the balancer, we use the fallback backends
     // returned by the resolver. Note that the fallback backend list may be
-    // empty, in which case the new round_robin policy will keep the requested
-    // picks pending.
+    // empty, in which case the new child policy will fail the picks.
     update_args.addresses = fallback_backend_addresses_;
+    if (fallback_backend_addresses_.ok() &&
+        fallback_backend_addresses_->empty()) {
+      update_args.resolution_note = absl::StrCat(
+          "grpclb in fallback mode without any balancer addresses: ",
+          resolution_note_);
+    }
   } else {
     update_args.addresses = serverlist_->GetServerAddressList(
         lb_calld_ == nullptr ? nullptr : lb_calld_->client_stats());
@@ -1674,6 +1710,57 @@ void GrpcLb::CreateOrUpdateChildPolicyLocked() {
   child_policy_->UpdateLocked(std::move(update_args));
 }
 
+//
+// subchannel caching
+//
+
+void GrpcLb::CacheDeletedSubchannelLocked(
+    RefCountedPtr<SubchannelInterface> subchannel) {
+  grpc_millis deletion_time =
+      ExecCtx::Get()->Now() + subchannel_cache_interval_ms_;
+  cached_subchannels_[deletion_time].push_back(std::move(subchannel));
+  if (!subchannel_cache_timer_pending_) {
+    Ref(DEBUG_LOCATION, "OnSubchannelCacheTimer").release();
+    subchannel_cache_timer_pending_ = true;
+    StartSubchannelCacheTimerLocked();
+  }
+}
+
+void GrpcLb::StartSubchannelCacheTimerLocked() {
+  GPR_ASSERT(!cached_subchannels_.empty());
+  grpc_timer_init(&subchannel_cache_timer_, cached_subchannels_.begin()->first,
+                  &on_subchannel_cache_timer_);
+}
+
+void GrpcLb::OnSubchannelCacheTimer(void* arg, grpc_error_handle error) {
+  auto* self = static_cast<GrpcLb*>(arg);
+  (void)GRPC_ERROR_REF(error);
+  self->work_serializer()->Run(
+      [self, error]() { self->GrpcLb::OnSubchannelCacheTimerLocked(error); },
+      DEBUG_LOCATION);
+}
+
+void GrpcLb::OnSubchannelCacheTimerLocked(grpc_error_handle error) {
+  if (subchannel_cache_timer_pending_ && error == GRPC_ERROR_NONE) {
+    auto it = cached_subchannels_.begin();
+    if (it != cached_subchannels_.end()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
+        gpr_log(GPR_INFO,
+                "[grpclb %p] removing %" PRIuPTR " subchannels from cache",
+                this, it->second.size());
+      }
+      cached_subchannels_.erase(it);
+    }
+    if (!cached_subchannels_.empty()) {
+      StartSubchannelCacheTimerLocked();
+      return;
+    }
+    subchannel_cache_timer_pending_ = false;
+  }
+  Unref(DEBUG_LOCATION, "OnSubchannelCacheTimer");
+  GRPC_ERROR_UNREF(error);
+}
+
 //
 // factory
 //
@@ -1744,39 +1831,34 @@ class GrpcLbFactory : public LoadBalancingPolicyFactory {
 // Plugin registration
 //
 
-namespace {
-
-// Only add client_load_reporting filter if the grpclb LB policy is used.
-bool maybe_add_client_load_reporting_filter(grpc_channel_stack_builder* builder,
-                                            void* arg) {
-  const grpc_channel_args* args =
-      grpc_channel_stack_builder_get_channel_arguments(builder);
-  const grpc_arg* channel_arg =
-      grpc_channel_args_find(args, GRPC_ARG_LB_POLICY_NAME);
-  if (channel_arg != nullptr && channel_arg->type == GRPC_ARG_STRING &&
-      strcmp(channel_arg->value.string, "grpclb") == 0) {
-    // TODO(roth): When we get around to re-attempting
-    // https://github.com/grpc/grpc/pull/16214, we should try to keep
-    // this filter at the very top of the subchannel stack, since that
-    // will minimize the number of metadata elements that the filter
-    // needs to iterate through to find the ClientStats object.
-    return grpc_channel_stack_builder_prepend_filter(
-        builder, static_cast<const grpc_channel_filter*>(arg), nullptr,
-        nullptr);
-  }
-  return true;
-}
-
-}  // namespace
-
 void grpc_lb_policy_grpclb_init() {
   grpc_core::LoadBalancingPolicyRegistry::Builder::
       RegisterLoadBalancingPolicyFactory(
           absl::make_unique<grpc_core::GrpcLbFactory>());
-  grpc_channel_init_register_stage(
-      GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY,
-      maybe_add_client_load_reporting_filter,
-      const_cast<grpc_channel_filter*>(&grpc_client_load_reporting_filter));
 }
 
 void grpc_lb_policy_grpclb_shutdown() {}
+
+namespace grpc_core {
+void RegisterGrpcLbLoadReportingFilter(CoreConfiguration::Builder* builder) {
+  builder->channel_init()->RegisterStage(
+      GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY,
+      [](grpc_channel_stack_builder* builder) {
+        const grpc_channel_args* args =
+            grpc_channel_stack_builder_get_channel_arguments(builder);
+        const grpc_arg* channel_arg =
+            grpc_channel_args_find(args, GRPC_ARG_LB_POLICY_NAME);
+        if (channel_arg != nullptr && channel_arg->type == GRPC_ARG_STRING &&
+            strcmp(channel_arg->value.string, "grpclb") == 0) {
+          // TODO(roth): When we get around to re-attempting
+          // https://github.com/grpc/grpc/pull/16214, we should try to keep
+          // this filter at the very top of the subchannel stack, since that
+          // will minimize the number of metadata elements that the filter
+          // needs to iterate through to find the ClientStats object.
+          return grpc_channel_stack_builder_prepend_filter(
+              builder, &grpc_client_load_reporting_filter, nullptr, nullptr);
+        }
+        return true;
+      });
+}
+}  // namespace grpc_core

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h

@@ -32,12 +32,9 @@
 #define GRPC_ARG_ADDRESS_IS_BACKEND_FROM_GRPCLB_LOAD_BALANCER \
   "grpc.address_is_backend_from_grpclb_load_balancer"
 
-namespace grpc_core {
-
-extern const char kGrpcLbClientStatsMetadataKey[];
-extern const char kGrpcLbLbTokenMetadataKey[];
-
-}  // namespace grpc_core
+// For use in tests.
+#define GRPC_ARG_GRPCLB_SUBCHANNEL_CACHE_INTERVAL_MS \
+  "grpc.internal.grpclb_subchannel_cache_interval_ms"
 
 #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_GRPCLB_GRPCLB_H \
         */

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.cc

@@ -42,7 +42,7 @@ int BalancerAddressesArgCmp(void* p, void* q) {
   ServerAddressList* address_list1 = static_cast<ServerAddressList*>(p);
   ServerAddressList* address_list2 = static_cast<ServerAddressList*>(q);
   if (address_list1 == nullptr || address_list2 == nullptr) {
-    return GPR_ICMP(address_list1, address_list2);
+    return QsortCompare(address_list1, address_list2);
   }
   if (address_list1->size() > address_list2->size()) return 1;
   if (address_list1->size() < address_list2->size()) return -1;

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.h

@@ -21,7 +21,7 @@
 
 #include <grpc/impl/codegen/grpc_types.h>
 
-#include "src/core/ext/filters/client_channel/server_address.h"
+#include "src/core/lib/resolver/server_address.h"
 
 namespace grpc_core {
 

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h

@@ -23,7 +23,7 @@
 
 #include <grpc/impl/codegen/grpc_types.h>
 
-#include "src/core/ext/filters/client_channel/server_address.h"
+#include "src/core/lib/resolver/server_address.h"
 
 namespace grpc_core {
 

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc

@@ -29,10 +29,10 @@
 #include "src/core/ext/filters/client_channel/client_channel.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h"
-#include "src/core/ext/filters/client_channel/server_address.h"
 #include "src/core/lib/address_utils/sockaddr_utils.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/gpr/string.h"
+#include "src/core/lib/resolver/server_address.h"
 #include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/slice/slice_internal.h"
 

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc

@@ -63,8 +63,7 @@ void GrpcLbClientStats::AddCallDropped(const char* token) {
     }
   }
   // Not found, so add a new entry.
-  drop_token_counts_->emplace_back(
-      grpc_core::UniquePtr<char>(gpr_strdup(token)), 1);
+  drop_token_counts_->emplace_back(UniquePtr<char>(gpr_strdup(token)), 1);
 }
 
 namespace {

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h

@@ -34,10 +34,10 @@ namespace grpc_core {
 class GrpcLbClientStats : public RefCounted<GrpcLbClientStats> {
  public:
   struct DropTokenCount {
-    grpc_core::UniquePtr<char> token;
+    UniquePtr<char> token;
     int64_t count;
 
-    DropTokenCount(grpc_core::UniquePtr<char> token, int64_t count)
+    DropTokenCount(UniquePtr<char> token, int64_t count)
         : token(std::move(token)), count(count) {}
   };