grpc 1.58.3 → 1.59.0

data/src/core/ext/filters/client_channel/client_channel.h

@@ -62,6 +62,8 @@
 #include "src/core/lib/iomgr/iomgr_fwd.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/load_balancing/lb_policy.h"
+#include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/arena_promise.h"
 #include "src/core/lib/resolver/resolver.h"
 #include "src/core/lib/resource_quota/arena.h"
 #include "src/core/lib/service_config/service_config.h"
@@ -102,10 +104,12 @@ namespace grpc_core {
 
 class ClientChannel {
  public:
-  static const grpc_channel_filter kFilterVtable;
+  static const grpc_channel_filter kFilterVtableWithPromises;
+  static const grpc_channel_filter kFilterVtableWithoutPromises;
 
   class LoadBalancedCall;
   class FilterBasedLoadBalancedCall;
+  class PromiseBasedLoadBalancedCall;
 
   // Flag that this object gets stored in channel args as a raw pointer.
   struct RawPointerChannelArgTag {};
@@ -115,6 +119,10 @@ class ClientChannel {
   // is not a client channel.
   static ClientChannel* GetFromChannel(Channel* channel);
 
+  static ArenaPromise<ServerMetadataHandle> MakeCallPromise(
+      grpc_channel_element* elem, CallArgs call_args,
+      NextPromiseFactory next_promise_factory);
+
   grpc_connectivity_state CheckConnectivityState(bool try_to_connect);
 
   // Starts a one-time connectivity state watch.  When the channel's state
@@ -164,6 +172,10 @@ class ClientChannel {
       grpc_closure* on_call_destruction_complete,
       absl::AnyInvocable<void()> on_commit, bool is_transparent_retry);
 
+  ArenaPromise<ServerMetadataHandle> CreateLoadBalancedCallPromise(
+      CallArgs call_args, absl::AnyInvocable<void()> on_commit,
+      bool is_transparent_retry);
+
   // Exposed for testing only.
   static ChannelArgs MakeSubchannelArgs(
       const ChannelArgs& channel_args, const ChannelArgs& address_args,
@@ -173,6 +185,7 @@ class ClientChannel {
  private:
   class CallData;
   class FilterBasedCallData;
+  class PromiseBasedCallData;
   class ResolverResultHandler;
   class SubchannelWrapper;
   class ClientChannelControlHelper;
@@ -315,8 +328,10 @@ class ClientChannel {
   mutable Mutex lb_mu_;
   RefCountedPtr<LoadBalancingPolicy::SubchannelPicker> picker_
       ABSL_GUARDED_BY(lb_mu_);
-  absl::flat_hash_set<LoadBalancedCall*> lb_queued_calls_
-      ABSL_GUARDED_BY(lb_mu_);
+  absl::flat_hash_set<RefCountedPtr<LoadBalancedCall>,
+                      RefCountedPtrHash<LoadBalancedCall>,
+                      RefCountedPtrEq<LoadBalancedCall>>
+      lb_queued_calls_ ABSL_GUARDED_BY(lb_mu_);
 
   //
   // Fields used in the control plane.  Guarded by work_serializer.
@@ -377,7 +392,7 @@ class ClientChannel::LoadBalancedCall
                    bool is_transparent_retry);
   ~LoadBalancedCall() override;
 
-  void Orphan() override;
+  void Orphan() override { Unref(); }
 
   // Called by channel when removing a call from the list of queued calls.
   void RemoveCallFromLbQueuedCallsLocked()
@@ -394,7 +409,6 @@ class ClientChannel::LoadBalancedCall
     return static_cast<ClientCallTracer::CallAttemptTracer*>(
         call_context()[GRPC_CONTEXT_CALL_TRACER].value);
   }
-  gpr_cycle_counter lb_call_start_time() const { return lb_call_start_time_; }
   ConnectedSubchannel* connected_subchannel() const {
     return connected_subchannel_.get();
   }
@@ -425,6 +439,8 @@ class ClientChannel::LoadBalancedCall
                             grpc_transport_stream_stats* transport_stream_stats,
                             absl::string_view peer_address);
 
+  void RecordLatency();
+
  private:
   class LbCallState;
   class Metadata;
@@ -432,7 +448,7 @@ class ClientChannel::LoadBalancedCall
 
   virtual Arena* arena() const = 0;
   virtual grpc_call_context_element* call_context() const = 0;
-  virtual grpc_polling_entity* pollent() const = 0;
+  virtual grpc_polling_entity* pollent() = 0;
   virtual grpc_metadata_batch* send_initial_metadata() const = 0;
 
   // Helper function for performing an LB pick with a specified picker.
@@ -445,7 +461,7 @@ class ClientChannel::LoadBalancedCall
 
   // Called when adding the call to the LB queue.
   virtual void OnAddToQueueLocked()
-      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&ClientChannel::lb_mu_) {}
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&ClientChannel::lb_mu_) = 0;
 
   ClientChannel* chand_;
 
@@ -496,7 +512,7 @@ class ClientChannel::FilterBasedLoadBalancedCall
   grpc_call_context_element* call_context() const override {
     return call_context_;
   }
-  grpc_polling_entity* pollent() const override { return pollent_; }
+  grpc_polling_entity* pollent() override { return pollent_; }
   grpc_metadata_batch* send_initial_metadata() const override {
     return pending_batches_[0]
         ->payload->send_initial_metadata.send_initial_metadata;
@@ -590,6 +606,34 @@ class ClientChannel::FilterBasedLoadBalancedCall
   grpc_transport_stream_op_batch* pending_batches_[MAX_PENDING_BATCHES] = {};
 };
 
+class ClientChannel::PromiseBasedLoadBalancedCall
+    : public ClientChannel::LoadBalancedCall {
+ public:
+  PromiseBasedLoadBalancedCall(ClientChannel* chand,
+                               absl::AnyInvocable<void()> on_commit,
+                               bool is_transparent_retry);
+
+  ArenaPromise<ServerMetadataHandle> MakeCallPromise(
+      CallArgs call_args, OrphanablePtr<PromiseBasedLoadBalancedCall> lb_call);
+
+ private:
+  Arena* arena() const override;
+  grpc_call_context_element* call_context() const override;
+  grpc_polling_entity* pollent() override { return &pollent_; }
+  grpc_metadata_batch* send_initial_metadata() const override;
+
+  void RetryPickLocked() override;
+
+  void OnAddToQueueLocked() override
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(&ClientChannel::lb_mu_);
+
+  grpc_polling_entity pollent_;
+  ClientMetadataHandle client_initial_metadata_;
+  Waker waker_;
+  bool was_queued_ = false;
+  Slice peer_string_;
+};
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_CLIENT_CHANNEL_H

data/src/core/ext/filters/client_channel/client_channel_plugin.cc

@@ -18,9 +18,14 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "absl/types/optional.h"
+
+#include <grpc/impl/channel_arg_names.h>
+
 #include "src/core/ext/filters/client_channel/client_channel.h"
 #include "src/core/ext/filters/client_channel/client_channel_service_config.h"
 #include "src/core/ext/filters/client_channel/retry_service_config.h"
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack_builder.h"
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/surface/channel_init.h"
@@ -28,13 +33,22 @@
 
 namespace grpc_core {
 
+namespace {
+bool IsEverythingBelowClientChannelPromiseSafe(const ChannelArgs& args) {
+  return !args.GetBool(GRPC_ARG_ENABLE_RETRIES).value_or(true);
+}
+}  // namespace
+
 void BuildClientChannelConfiguration(CoreConfiguration::Builder* builder) {
   internal::ClientChannelServiceConfigParser::Register(builder);
   internal::RetryServiceConfigParser::Register(builder);
   builder->channel_init()->RegisterStage(
       GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY,
       [](ChannelStackBuilder* builder) {
-        builder->AppendFilter(&ClientChannel::kFilterVtable);
+        builder->AppendFilter(
+            IsEverythingBelowClientChannelPromiseSafe(builder->channel_args())
+                ? &ClientChannel::kFilterVtableWithPromises
+                : &ClientChannel::kFilterVtableWithoutPromises);
         return true;
       });
 }

data/src/core/ext/filters/client_channel/dynamic_filters.h

@@ -99,6 +99,8 @@ class DynamicFilters : public RefCounted<DynamicFilters> {
 
   RefCountedPtr<Call> CreateCall(Call::Args args, grpc_error_handle* error);
 
+  grpc_channel_stack* channel_stack() const { return channel_stack_.get(); }
+
  private:
   RefCountedPtr<grpc_channel_stack> channel_stack_;
 };

data/src/core/ext/filters/client_channel/{http_proxy.cc → http_proxy_mapper.cc}

@@ -18,7 +18,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/ext/filters/client_channel/http_proxy.h"
+#include "src/core/ext/filters/client_channel/http_proxy_mapper.h"
 
 #include <stdint.h>
 #include <string.h>

data/src/core/ext/filters/client_channel/{http_proxy.h → http_proxy_mapper.h}

@@ -16,8 +16,8 @@
 //
 //
 
-#ifndef GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_HTTP_PROXY_H
-#define GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_HTTP_PROXY_H
+#ifndef GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_HTTP_PROXY_MAPPER_H
+#define GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_HTTP_PROXY_MAPPER_H
 
 #include <grpc/support/port_platform.h>
 
@@ -49,4 +49,4 @@ void RegisterHttpProxyMapper(CoreConfiguration::Builder* builder);
 
 }  // namespace grpc_core
 
-#endif  // GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_HTTP_PROXY_H
+#endif  // GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_HTTP_PROXY_MAPPER_H

data/src/core/ext/filters/client_channel/lb_policy/endpoint_list.cc

@@ -0,0 +1,188 @@
+//
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/ext/filters/client_channel/lb_policy/endpoint_list.h"
+
+#include <stdlib.h>
+
+#include <algorithm>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "absl/types/optional.h"
+
+#include <grpc/impl/connectivity_state.h>
+#include <grpc/support/json.h>
+#include <grpc/support/log.h>
+
+#include "src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/config/core_configuration.h"
+#include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/iomgr/pollset_set.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/load_balancing/delegating_helper.h"
+#include "src/core/lib/load_balancing/lb_policy.h"
+#include "src/core/lib/load_balancing/lb_policy_registry.h"
+#include "src/core/lib/resolver/server_address.h"
+
+namespace grpc_core {
+
+//
+// EndpointList::Endpoint::Helper
+//
+
+class EndpointList::Endpoint::Helper
+    : public LoadBalancingPolicy::DelegatingChannelControlHelper {
+ public:
+  explicit Helper(RefCountedPtr<Endpoint> endpoint)
+      : endpoint_(std::move(endpoint)) {}
+
+  ~Helper() override { endpoint_.reset(DEBUG_LOCATION, "Helper"); }
+
+  RefCountedPtr<SubchannelInterface> CreateSubchannel(
+      ServerAddress address, const ChannelArgs& args) override {
+    return endpoint_->CreateSubchannel(std::move(address), args);
+  }
+
+  void UpdateState(
+      grpc_connectivity_state state, const absl::Status& status,
+      RefCountedPtr<LoadBalancingPolicy::SubchannelPicker> picker) override {
+    auto old_state = std::exchange(endpoint_->connectivity_state_, state);
+    endpoint_->picker_ = std::move(picker);
+    endpoint_->OnStateUpdate(old_state, state, status);
+  }
+
+ private:
+  LoadBalancingPolicy::ChannelControlHelper* parent_helper() const override {
+    return endpoint_->endpoint_list_->channel_control_helper();
+  }
+
+  RefCountedPtr<Endpoint> endpoint_;
+};
+
+//
+// EndpointList::Endpoint
+//
+
+void EndpointList::Endpoint::Init(
+    const ServerAddress& address, const ChannelArgs& args,
+    std::shared_ptr<WorkSerializer> work_serializer) {
+  ChannelArgs child_args =
+      args.Set(GRPC_ARG_INTERNAL_PICK_FIRST_ENABLE_HEALTH_CHECKING, true)
+          .Set(GRPC_ARG_INTERNAL_PICK_FIRST_OMIT_STATUS_MESSAGE_PREFIX, true);
+  LoadBalancingPolicy::Args lb_policy_args;
+  lb_policy_args.work_serializer = std::move(work_serializer);
+  lb_policy_args.args = child_args;
+  lb_policy_args.channel_control_helper =
+      std::make_unique<Helper>(Ref(DEBUG_LOCATION, "Helper"));
+  child_policy_ =
+      CoreConfiguration::Get().lb_policy_registry().CreateLoadBalancingPolicy(
+          "pick_first", std::move(lb_policy_args));
+  if (GPR_UNLIKELY(endpoint_list_->tracer_ != nullptr)) {
+    gpr_log(GPR_INFO, "[%s %p] endpoint %p: created child policy %p",
+            endpoint_list_->tracer_, endpoint_list_->policy_.get(), this,
+            child_policy_.get());
+  }
+  // Add our interested_parties pollset_set to that of the newly created
+  // child policy. This will make the child policy progress upon activity on
+  // this policy, which in turn is tied to the application's call.
+  grpc_pollset_set_add_pollset_set(
+      child_policy_->interested_parties(),
+      endpoint_list_->policy_->interested_parties());
+  // Construct pick_first config.
+  auto config =
+      CoreConfiguration::Get().lb_policy_registry().ParseLoadBalancingConfig(
+          Json::FromArray(
+              {Json::FromObject({{"pick_first", Json::FromObject({})}})}));
+  GPR_ASSERT(config.ok());
+  // Update child policy.
+  LoadBalancingPolicy::UpdateArgs update_args;
+  update_args.addresses.emplace().emplace_back(address);
+  update_args.args = child_args;
+  update_args.config = std::move(*config);
+  // TODO(roth): If the child reports a non-OK status with the update,
+  // we need to propagate that back to the resolver somehow.
+  (void)child_policy_->UpdateLocked(std::move(update_args));
+}
+
+void EndpointList::Endpoint::Orphan() {
+  // Remove pollset_set linkage.
+  grpc_pollset_set_del_pollset_set(
+      child_policy_->interested_parties(),
+      endpoint_list_->policy_->interested_parties());
+  child_policy_.reset();
+  picker_.reset();
+  Unref();
+}
+
+void EndpointList::Endpoint::ResetBackoffLocked() {
+  if (child_policy_ != nullptr) child_policy_->ResetBackoffLocked();
+}
+
+void EndpointList::Endpoint::ExitIdleLocked() {
+  if (child_policy_ != nullptr) child_policy_->ExitIdleLocked();
+}
+
+size_t EndpointList::Endpoint::Index() const {
+  for (size_t i = 0; i < endpoint_list_->endpoints_.size(); ++i) {
+    if (endpoint_list_->endpoints_[i].get() == this) return i;
+  }
+  return -1;
+}
+
+RefCountedPtr<SubchannelInterface> EndpointList::Endpoint::CreateSubchannel(
+    ServerAddress address, const ChannelArgs& args) {
+  return endpoint_list_->channel_control_helper()->CreateSubchannel(
+      std::move(address), args);
+}
+
+//
+// EndpointList
+//
+
+void EndpointList::Init(
+    const ServerAddressList& addresses, const ChannelArgs& args,
+    absl::AnyInvocable<OrphanablePtr<Endpoint>(
+        RefCountedPtr<EndpointList>, const ServerAddress&, const ChannelArgs&)>
+        create_endpoint) {
+  for (const ServerAddress& address : addresses) {
+    endpoints_.push_back(
+        create_endpoint(Ref(DEBUG_LOCATION, "Endpoint"), address, args));
+  }
+}
+
+void EndpointList::ResetBackoffLocked() {
+  for (const auto& endpoint : endpoints_) {
+    endpoint->ResetBackoffLocked();
+  }
+}
+
+bool EndpointList::AllEndpointsSeenInitialState() const {
+  for (const auto& endpoint : endpoints_) {
+    if (!endpoint->connectivity_state().has_value()) return false;
+  }
+  return true;
+}
+
+}  // namespace grpc_core

data/src/core/ext/filters/client_channel/lb_policy/endpoint_list.h

@@ -0,0 +1,214 @@
+//
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_ENDPOINT_LIST_H
+#define GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_ENDPOINT_LIST_H
+
+#include <grpc/support/port_platform.h>
+
+#include <stdlib.h>
+
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "absl/functional/any_invocable.h"
+#include "absl/status/status.h"
+#include "absl/types/optional.h"
+
+#include <grpc/impl/connectivity_state.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/gprpp/work_serializer.h"
+#include "src/core/lib/load_balancing/lb_policy.h"
+#include "src/core/lib/load_balancing/subchannel_interface.h"
+#include "src/core/lib/resolver/server_address.h"
+
+namespace grpc_core {
+
+// A list of endpoints for use in a petiole LB policy.  Each endpoint may
+// have one or more addresses, which will be passed down to a pick_first
+// child policy.
+//
+// To use this, a petiole policy must define its own subclass of both
+// EndpointList and EndpointList::Endpoint, like so:
+/*
+class MyEndpointList : public EndpointList {
+ public:
+  MyEndpointList(RefCountedPtr<MyLbPolicy> lb_policy,
+                 const ServerAddressList& addresses, const ChannelArgs& args)
+      : EndpointList(std::move(lb_policy),
+                     GRPC_TRACE_FLAG_ENABLED(grpc_my_tracer)
+                         ? "MyEndpointList"
+                         : nullptr) {
+    Init(addresses, args,
+         [&](RefCountedPtr<MyEndpointList> endpoint_list,
+             const ServerAddress& address, const ChannelArgs& args) {
+           return MakeOrphanable<MyEndpoint>(
+               std::move(endpoint_list), address, args,
+               policy<MyLbPolicy>()->work_serializer());
+         });
+  }
+
+ private:
+  class MyEndpoint : public Endpoint {
+   public:
+    MyEndpoint(RefCountedPtr<MyEndpointList> endpoint_list,
+               const ServerAddress& address, const ChannelArgs& args,
+               std::shared_ptr<WorkSerializer> work_serializer)
+        : Endpoint(std::move(endpoint_list)) {
+      Init(address, args, std::move(work_serializer));
+    }
+
+   private:
+    void OnStateUpdate(
+        absl::optional<grpc_connectivity_state> old_state,
+        grpc_connectivity_state new_state,
+        const absl::Status& status) override {
+      // ...handle connectivity state change...
+    }
+  };
+
+  LoadBalancingPolicy::ChannelControlHelper* channel_control_helper()
+      const override {
+    return policy<MyLbPolicy>()->channel_control_helper();
+  }
+};
+*/
+// TODO(roth): Consider wrapping this in an LB policy subclass for petiole
+// policies to inherit from.
+class EndpointList : public InternallyRefCounted<EndpointList> {
+ public:
+  // An individual endpoint.
+  class Endpoint : public InternallyRefCounted<Endpoint> {
+   public:
+    ~Endpoint() override { endpoint_list_.reset(DEBUG_LOCATION, "Endpoint"); }
+
+    void Orphan() override;
+
+    void ResetBackoffLocked();
+    void ExitIdleLocked();
+
+    absl::optional<grpc_connectivity_state> connectivity_state() const {
+      return connectivity_state_;
+    }
+    RefCountedPtr<LoadBalancingPolicy::SubchannelPicker> picker() const {
+      return picker_;
+    }
+
+   protected:
+    // We use two-phase initialization here to ensure that the vtable is
+    // initialized before we need to use it.  Subclass must invoke Init()
+    // from inside its ctor.
+    explicit Endpoint(RefCountedPtr<EndpointList> endpoint_list)
+        : endpoint_list_(std::move(endpoint_list)) {}
+
+    void Init(const ServerAddress& address, const ChannelArgs& args,
+              std::shared_ptr<WorkSerializer> work_serializer);
+
+    // Templated for convenience, to provide a short-hand for
+    // down-casting in the caller.
+    template <typename T>
+    T* endpoint_list() const {
+      return static_cast<T*>(endpoint_list_.get());
+    }
+
+    // Templated for convenience, to provide a short-hand for down-casting
+    // in the caller.
+    template <typename T>
+    T* policy() const {
+      return endpoint_list_->policy<T>();
+    }
+
+    // Returns the index of this endpoint within the EndpointList.
+    // Intended for trace logging.
+    size_t Index() const;
+
+   private:
+    class Helper;
+
+    // Called when the child policy reports a connectivity state update.
+    virtual void OnStateUpdate(
+        absl::optional<grpc_connectivity_state> old_state,
+        grpc_connectivity_state new_state, const absl::Status& status) = 0;
+
+    // Called to create a subchannel.  Subclasses may override.
+    virtual RefCountedPtr<SubchannelInterface> CreateSubchannel(
+        ServerAddress address, const ChannelArgs& args);
+
+    RefCountedPtr<EndpointList> endpoint_list_;
+
+    OrphanablePtr<LoadBalancingPolicy> child_policy_;
+    absl::optional<grpc_connectivity_state> connectivity_state_;
+    RefCountedPtr<LoadBalancingPolicy::SubchannelPicker> picker_;
+  };
+
+  ~EndpointList() override { policy_.reset(DEBUG_LOCATION, "EndpointList"); }
+
+  void Orphan() override {
+    endpoints_.clear();
+    Unref();
+  }
+
+  size_t size() const { return endpoints_.size(); }
+
+  const std::vector<OrphanablePtr<Endpoint>>& endpoints() const {
+    return endpoints_;
+  }
+
+  void ResetBackoffLocked();
+
+ protected:
+  // We use two-phase initialization here to ensure that the vtable is
+  // initialized before we need to use it.  Subclass must invoke Init()
+  // from inside its ctor.
+  EndpointList(RefCountedPtr<LoadBalancingPolicy> policy, const char* tracer)
+      : policy_(std::move(policy)), tracer_(tracer) {}
+
+  void Init(const ServerAddressList& addresses, const ChannelArgs& args,
+            absl::AnyInvocable<OrphanablePtr<Endpoint>(
+                RefCountedPtr<EndpointList>, const ServerAddress&,
+                const ChannelArgs&)>
+                create_endpoint);
+
+  // Templated for convenience, to provide a short-hand for down-casting
+  // in the caller.
+  template <typename T>
+  T* policy() const {
+    return static_cast<T*>(policy_.get());
+  }
+
+  // Returns true if all endpoints have seen their initial connectivity
+  // state notification.
+  bool AllEndpointsSeenInitialState() const;
+
+ private:
+  // Returns the parent policy's helper.  Needed because the accessor
+  // method is protected on LoadBalancingPolicy.
+  virtual LoadBalancingPolicy::ChannelControlHelper* channel_control_helper()
+      const = 0;
+
+  RefCountedPtr<LoadBalancingPolicy> policy_;
+  const char* tracer_;
+  std::vector<OrphanablePtr<Endpoint>> endpoints_;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_ENDPOINT_LIST_H

data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc

@@ -107,6 +107,7 @@
 #include "src/core/lib/channel/channelz.h"
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/crash.h"
@@ -318,10 +319,22 @@ class GrpcLb : public LoadBalancingPolicy {
           lb_token_(std::move(lb_token)),
           client_stats_(std::move(client_stats)) {}
 
-    ~SubchannelWrapper() override {
-      if (!lb_policy_->shutting_down_) {
-        lb_policy_->CacheDeletedSubchannelLocked(wrapped_subchannel());
+    void Orphan() override {
+      if (!IsWorkSerializerDispatchEnabled()) {
+        if (!lb_policy_->shutting_down_) {
+          lb_policy_->CacheDeletedSubchannelLocked(wrapped_subchannel());
+        }
+        return;
       }
+      WeakRefCountedPtr<SubchannelWrapper> self = WeakRef();
+      lb_policy_->work_serializer()->Run(
+          [self = std::move(self)]() {
+            if (!self->lb_policy_->shutting_down_) {
+              self->lb_policy_->CacheDeletedSubchannelLocked(
+                  self->wrapped_subchannel());
+            }
+          },
+          DEBUG_LOCATION);
     }
 
     const std::string& lb_token() const { return lb_token_; }
@@ -1588,7 +1601,7 @@ absl::Status GrpcLb::UpdateBalancerChannelLocked() {
   // Pass channel creds via channel args, since the fake resolver won't
   // do this automatically.
   result.args = lb_channel_args.SetObject(std::move(channel_credentials));
-  response_generator_->SetResponse(std::move(result));
+  response_generator_->SetResponseAsync(std::move(result));
   // Return status.
   return status;
 }