grpc 1.23.0 → 1.24.0

data/include/grpc/grpc.h

@@ -224,7 +224,9 @@ GRPCAPI grpc_call* grpc_channel_create_call(
 GRPCAPI void grpc_channel_ping(grpc_channel* channel, grpc_completion_queue* cq,
                                void* tag, void* reserved);
 
-/** Pre-register a method/host pair on a channel. */
+/** Pre-register a method/host pair on a channel.
+    method and host are not owned and must remain alive while the server is
+    running. */
 GRPCAPI void* grpc_channel_register_call(grpc_channel* channel,
                                          const char* method, const char* host,
                                          void* reserved);

data/include/grpc/grpc_security.h

@@ -778,6 +778,21 @@ GRPCAPI int grpc_tls_key_materials_config_set_key_materials(
     const grpc_ssl_pem_key_cert_pair** pem_key_cert_pairs,
     size_t num_key_cert_pairs);
 
+/** Set grpc_tls_key_materials_config instance with a provided version number,
+    which is used to keep track of the version of key materials.
+    It returns 1 on success and 0 on failure. It is used for
+    experimental purpose for now and subject to change.
+ */
+GRPCAPI int grpc_tls_key_materials_config_set_version(
+    grpc_tls_key_materials_config* config, int version);
+
+/** Get the version number of a grpc_tls_key_materials_config instance.
+    It returns the version number on success and -1 on failure.
+    It is used for experimental purpose for now and subject to change.
+ */
+GRPCAPI int grpc_tls_key_materials_config_get_version(
+    grpc_tls_key_materials_config* config);
+
 /** --- TLS credential reload config. ---
     It is used for experimental purpose for now and subject to change.*/
 
@@ -793,10 +808,11 @@ typedef void (*grpc_tls_on_credential_reload_done_cb)(
 /** A struct containing all information necessary to schedule/cancel
     a credential reload request. cb and cb_user_data represent a gRPC-provided
     callback and an argument passed to it. key_materials is an in/output
-    parameter containing currently used/newly reloaded credentials. status and
-    error_details are used to hold information about errors occurred when a
-    credential reload request is scheduled/cancelled. It is used for
-    experimental purpose for now and subject to change. */
+    parameter containing currently used/newly reloaded credentials. If
+    credential reload does not result in a new credential, key_materials should
+    not be modified. status and error_details are used to hold information about
+    errors occurred when a credential reload request is scheduled/cancelled. It
+    is used for experimental purpose for now and subject to change. */
 struct grpc_tls_credential_reload_arg {
   grpc_tls_on_credential_reload_done_cb cb;
   void* cb_user_data;

data/include/grpc/impl/codegen/grpc_types.h

@@ -169,11 +169,7 @@ typedef struct {
 #define GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS "grpc.max_connection_age_grace_ms"
 /** Timeout after the last RPC finishes on the client channel at which the
  * channel goes back into IDLE state. Int valued, milliseconds. INT_MAX means
- * unlimited. */
-/** TODO(qianchengz): Currently the default value is INT_MAX, which means the
- * client idle filter is disabled by default. After the client idle filter
- * proves no perfomance issue, we will change the default value to a reasonable
- * value. */
+ * unlimited. The default value is 30 minutes and the min value is 1 second. */
 #define GRPC_ARG_CLIENT_IDLE_TIMEOUT_MS "grpc.client_idle_timeout_ms"
 /** Enable/disable support for per-message compression. Defaults to 1, unless
     GRPC_ARG_MINIMAL_STACK is enabled, in which case it defaults to 0. */
@@ -330,6 +326,11 @@ typedef struct {
    balancer before using fallback backend addresses from the resolver.
    If 0, enter fallback mode immediately. Default value is 10000. */
 #define GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS "grpc.xds_fallback_timeout_ms"
+/* Time in milliseconds to wait before a locality is deleted after it's removed
+   from the received EDS update. If 0, delete the locality immediately. Default
+   value is 15 minutes. */
+#define GRPC_ARG_LOCALITY_RETENTION_INTERVAL_MS \
+  "grpc.xds_locality_retention_interval_ms"
 /** If non-zero, grpc server's cronet compression workaround will be enabled */
 #define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \
   "grpc.workaround.cronet_compression"

data/include/grpc/impl/codegen/port_platform.h

@@ -27,6 +27,14 @@
  *  - some syscalls to be made directly
  */
 
+/*
+ * Defines GRPC_USE_CPP_STD_LIB to use standard C++ library instead of
+ * in-house library if possible. (e.g. std::map)
+ */
+#ifndef GRPC_USE_CPP_STD_LIB
+#define GRPC_USE_CPP_STD_LIB 1
+#endif
+
 /* Get windows.h included everywhere (we need it) */
 #if defined(_WIN64) || defined(WIN64) || defined(_WIN32) || defined(WIN32)
 #ifndef WIN32_LEAN_AND_MEAN
@@ -463,6 +471,23 @@ typedef unsigned __int64 uint64_t;
 #include <stdint.h>
 #endif /* _MSC_VER */
 
+/* Type of cycle clock implementation */
+#ifdef GPR_LINUX
+/* Disable cycle clock by default.
+   TODO(soheil): enable when we support fallback for unstable cycle clocks.
+#if defined(__i386__)
+#define GPR_CYCLE_COUNTER_RDTSC_32 1
+#elif defined(__x86_64__) || defined(__amd64__)
+#define GPR_CYCLE_COUNTER_RDTSC_64 1
+#else
+#define GPR_CYCLE_COUNTER_FALLBACK 1
+#endif
+*/
+#define GPR_CYCLE_COUNTER_FALLBACK 1
+#else
+#define GPR_CYCLE_COUNTER_FALLBACK 1
+#endif /* GPR_LINUX */
+
 /* Cache line alignment */
 #ifndef GPR_CACHELINE_SIZE_LOG
 #if defined(__i386__) || defined(__x86_64__)

data/src/core/ext/filters/client_channel/backend_metric.cc

@@ -0,0 +1,78 @@
+//
+// Copyright 2019 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/ext/filters/client_channel/backend_metric.h"
+
+#include "src/core/lib/gprpp/string_view.h"
+#include "udpa/data/orca/v1/orca_load_report.upb.h"
+
+namespace grpc_core {
+
+namespace {
+
+template <typename EntryType>
+Map<StringView, double, StringLess> ParseMap(
+    udpa_data_orca_v1_OrcaLoadReport* msg,
+    EntryType** (*entry_func)(udpa_data_orca_v1_OrcaLoadReport*, size_t*),
+    upb_strview (*key_func)(const EntryType*),
+    double (*value_func)(const EntryType*), Arena* arena) {
+  Map<StringView, double, StringLess> result;
+  size_t size;
+  const auto* const* entries = entry_func(msg, &size);
+  for (size_t i = 0; i < size; ++i) {
+    upb_strview key_view = key_func(entries[i]);
+    char* key = static_cast<char*>(arena->Alloc(key_view.size + 1));
+    memcpy(key, key_view.data, key_view.size);
+    result[StringView(key, key_view.size)] = value_func(entries[i]);
+  }
+  return result;
+}
+
+}  // namespace
+
+const LoadBalancingPolicy::BackendMetricData* ParseBackendMetricData(
+    const grpc_slice& serialized_load_report, Arena* arena) {
+  upb::Arena upb_arena;
+  udpa_data_orca_v1_OrcaLoadReport* msg =
+      udpa_data_orca_v1_OrcaLoadReport_parse(
+          reinterpret_cast<const char*>(
+              GRPC_SLICE_START_PTR(serialized_load_report)),
+          GRPC_SLICE_LENGTH(serialized_load_report), upb_arena.ptr());
+  if (msg == nullptr) return nullptr;
+  LoadBalancingPolicy::BackendMetricData* backend_metric_data =
+      arena->New<LoadBalancingPolicy::BackendMetricData>();
+  backend_metric_data->cpu_utilization =
+      udpa_data_orca_v1_OrcaLoadReport_cpu_utilization(msg);
+  backend_metric_data->mem_utilization =
+      udpa_data_orca_v1_OrcaLoadReport_mem_utilization(msg);
+  backend_metric_data->requests_per_second =
+      udpa_data_orca_v1_OrcaLoadReport_rps(msg);
+  backend_metric_data->request_cost =
+      ParseMap<udpa_data_orca_v1_OrcaLoadReport_RequestCostEntry>(
+          msg, udpa_data_orca_v1_OrcaLoadReport_mutable_request_cost,
+          udpa_data_orca_v1_OrcaLoadReport_RequestCostEntry_key,
+          udpa_data_orca_v1_OrcaLoadReport_RequestCostEntry_value, arena);
+  backend_metric_data->utilization =
+      ParseMap<udpa_data_orca_v1_OrcaLoadReport_UtilizationEntry>(
+          msg, udpa_data_orca_v1_OrcaLoadReport_mutable_utilization,
+          udpa_data_orca_v1_OrcaLoadReport_UtilizationEntry_key,
+          udpa_data_orca_v1_OrcaLoadReport_UtilizationEntry_value, arena);
+  return backend_metric_data;
+}
+
+}  // namespace grpc_core

data/src/core/ext/filters/client_channel/backend_metric.h

@@ -0,0 +1,36 @@
+//
+// Copyright 2019 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_BACKEND_METRIC_H
+#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_BACKEND_METRIC_H
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/slice.h>
+
+#include "src/core/ext/filters/client_channel/lb_policy.h"
+#include "src/core/lib/gprpp/arena.h"
+
+namespace grpc_core {
+
+// Parses the serialized load report and allocates a BackendMetricData
+// object on the arena.
+const LoadBalancingPolicy::BackendMetricData* ParseBackendMetricData(
+    const grpc_slice& serialized_load_report, Arena* arena);
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_BACKEND_METRIC_H */

data/src/core/ext/filters/client_channel/channel_connectivity.cc

@@ -111,6 +111,12 @@ static void finished_completion(void* pw, grpc_cq_completion* ignored) {
 
 static void partly_done(state_watcher* w, bool due_to_completion,
                         grpc_error* error) {
+  bool end_op = false;
+  void* end_op_tag = nullptr;
+  grpc_error* end_op_error = nullptr;
+  grpc_completion_queue* end_op_cq = nullptr;
+  grpc_cq_completion* end_op_completion_storage = nullptr;
+
   if (due_to_completion) {
     grpc_timer_cancel(&w->alarm);
   } else {
@@ -152,8 +158,11 @@ static void partly_done(state_watcher* w, bool due_to_completion,
         w->error = error;
       }
       w->phase = CALLING_BACK_AND_FINISHED;
-      grpc_cq_end_op(w->cq, w->tag, w->error, finished_completion, w,
-                     &w->completion_storage);
+      end_op = true;
+      end_op_cq = w->cq;
+      end_op_tag = w->tag;
+      end_op_error = w->error;
+      end_op_completion_storage = &w->completion_storage;
       break;
     case CALLING_BACK_AND_FINISHED:
       GPR_UNREACHABLE_CODE(return );
@@ -161,6 +170,11 @@ static void partly_done(state_watcher* w, bool due_to_completion,
   }
   gpr_mu_unlock(&w->mu);
 
+  if (end_op) {
+    grpc_cq_end_op(end_op_cq, end_op_tag, end_op_error, finished_completion, w,
+                   end_op_completion_storage);
+  }
+
   GRPC_ERROR_UNREF(error);
 }
 

data/src/core/ext/filters/client_channel/client_channel.cc

@@ -31,6 +31,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/sync.h>
 
+#include "src/core/ext/filters/client_channel/backend_metric.h"
 #include "src/core/ext/filters/client_channel/backup_poller.h"
 #include "src/core/ext/filters/client_channel/global_subchannel_pool.h"
 #include "src/core/ext/filters/client_channel/http_connect_handshaker.h"
@@ -129,7 +130,7 @@ class ChannelData {
     return disconnect_error_.Load(MemoryOrder::ACQUIRE);
   }
 
-  grpc_combiner* data_plane_combiner() const { return data_plane_combiner_; }
+  Mutex* data_plane_mu() const { return &data_plane_mu_; }
 
   LoadBalancingPolicy::SubchannelPicker* picker() const {
     return picker_.get();
@@ -165,8 +166,6 @@ class ChannelData {
 
  private:
   class SubchannelWrapper;
-  class ConnectivityStateAndPickerSetter;
-  class ServiceConfigSetter;
   class ClientChannelControlHelper;
 
   class ExternalConnectivityWatcher {
@@ -213,6 +212,14 @@ class ChannelData {
   ChannelData(grpc_channel_element_args* args, grpc_error** error);
   ~ChannelData();
 
+  void UpdateStateAndPickerLocked(
+      grpc_connectivity_state state, const char* reason,
+      UniquePtr<LoadBalancingPolicy::SubchannelPicker> picker);
+
+  void UpdateServiceConfigLocked(
+      RefCountedPtr<ServerRetryThrottleData> retry_throttle_data,
+      RefCountedPtr<ServiceConfig> service_config);
+
   void CreateResolvingLoadBalancingPolicyLocked();
 
   void DestroyResolvingLoadBalancingPolicyLocked();
@@ -249,9 +256,9 @@ class ChannelData {
   channelz::ChannelNode* channelz_node_;
 
   //
-  // Fields used in the data plane.  Guarded by data_plane_combiner.
+  // Fields used in the data plane.  Guarded by data_plane_mu.
   //
-  grpc_combiner* data_plane_combiner_;
+  mutable Mutex data_plane_mu_;
   UniquePtr<LoadBalancingPolicy::SubchannelPicker> picker_;
   QueuedPick* queued_picks_ = nullptr;  // Linked list of queued picks.
   // Data from service config.
@@ -273,15 +280,21 @@ class ChannelData {
   bool received_first_resolver_result_ = false;
   // The number of SubchannelWrapper instances referencing a given Subchannel.
   Map<Subchannel*, int> subchannel_refcount_map_;
+  // The set of SubchannelWrappers that currently exist.
+  // No need to hold a ref, since the map is updated in the control-plane
+  // combiner when the SubchannelWrappers are created and destroyed.
+  // TODO(roth): We really want to use a set here, not a map.  Since we don't
+  // currently have a set implementation, we use a map and ignore the value.
+  Map<SubchannelWrapper*, bool> subchannel_wrappers_;
   // Pending ConnectedSubchannel updates for each SubchannelWrapper.
   // Updates are queued here in the control plane combiner and then applied
-  // in the data plane combiner when the picker is updated.
+  // in the data plane mutex when the picker is updated.
   Map<RefCountedPtr<SubchannelWrapper>, RefCountedPtr<ConnectedSubchannel>,
       RefCountedPtrLess<SubchannelWrapper>>
       pending_subchannel_updates_;
 
   //
-  // Fields accessed from both data plane and control plane combiners.
+  // Fields accessed from both data plane mutex and control plane combiner.
   //
   Atomic<grpc_error*> disconnect_error_;
 
@@ -315,7 +328,16 @@ class CallData {
   void MaybeApplyServiceConfigToCallLocked(grpc_call_element* elem);
 
   // Invoked by channel for queued picks when the picker is updated.
-  static void StartPickLocked(void* arg, grpc_error* error);
+  static void PickSubchannel(void* arg, grpc_error* error);
+
+  // Helper function for performing a pick while holding the data plane
+  // mutex.  Returns true if the pick is complete, in which case the caller
+  // must invoke PickDone() or AsyncPickDone() with the returned error.
+  bool PickSubchannelLocked(grpc_call_element* elem, grpc_error** error);
+
+  // Schedules a callback to process the completed pick.  The callback
+  // will not run until after this method returns.
+  void AsyncPickDone(grpc_call_element* elem, grpc_error* error);
 
  private:
   class QueuedPickCanceller;
@@ -374,6 +396,19 @@ class CallData {
 
     void* Alloc(size_t size) override { return calld_->arena_->Alloc(size); }
 
+    const LoadBalancingPolicy::BackendMetricData* GetBackendMetricData()
+        override {
+      if (calld_->backend_metric_data_ == nullptr) {
+        grpc_linked_mdelem* md = calld_->recv_trailing_metadata_->idx.named
+                                     .x_endpoint_load_metrics_bin;
+        if (md != nullptr) {
+          calld_->backend_metric_data_ =
+              ParseBackendMetricData(GRPC_MDVALUE(md->md), calld_->arena_);
+        }
+      }
+      return calld_->backend_metric_data_;
+    }
+
    private:
     CallData* calld_;
   };
@@ -685,7 +720,7 @@ class CallData {
   grpc_deadline_state deadline_state_;
 
   grpc_slice path_;  // Request path.
-  gpr_timespec call_start_time_;
+  gpr_cycle_counter call_start_time_;
   grpc_millis deadline_;
   Arena* arena_;
   grpc_call_stack* owning_call_;
@@ -706,9 +741,10 @@ class CallData {
   bool service_config_applied_ = false;
   QueuedPickCanceller* pick_canceller_ = nullptr;
   LbCallState lb_call_state_;
+  const LoadBalancingPolicy::BackendMetricData* backend_metric_data_ = nullptr;
   RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
   void (*lb_recv_trailing_metadata_ready_)(
-      void* user_data,
+      void* user_data, grpc_error* error,
       LoadBalancingPolicy::MetadataInterface* recv_trailing_metadata,
       LoadBalancingPolicy::CallState* call_state) = nullptr;
   void* lb_recv_trailing_metadata_ready_user_data_ = nullptr;
@@ -799,14 +835,14 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
     GRPC_CHANNEL_STACK_REF(chand_->owning_stack_, "SubchannelWrapper");
     auto* subchannel_node = subchannel_->channelz_node();
     if (subchannel_node != nullptr) {
-      intptr_t subchannel_uuid = subchannel_node->uuid();
       auto it = chand_->subchannel_refcount_map_.find(subchannel_);
       if (it == chand_->subchannel_refcount_map_.end()) {
-        chand_->channelz_node_->AddChildSubchannel(subchannel_uuid);
+        chand_->channelz_node_->AddChildSubchannel(subchannel_node->uuid());
         it = chand_->subchannel_refcount_map_.emplace(subchannel_, 0).first;
       }
       ++it->second;
     }
+    chand_->subchannel_wrappers_[this] = true;
   }
 
   ~SubchannelWrapper() {
@@ -815,14 +851,14 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
               "chand=%p: destroying subchannel wrapper %p for subchannel %p",
               chand_, this, subchannel_);
     }
+    chand_->subchannel_wrappers_.erase(this);
     auto* subchannel_node = subchannel_->channelz_node();
     if (subchannel_node != nullptr) {
-      intptr_t subchannel_uuid = subchannel_node->uuid();
       auto it = chand_->subchannel_refcount_map_.find(subchannel_);
       GPR_ASSERT(it != chand_->subchannel_refcount_map_.end());
       --it->second;
       if (it->second == 0) {
-        chand_->channelz_node_->RemoveChildSubchannel(subchannel_uuid);
+        chand_->channelz_node_->RemoveChildSubchannel(subchannel_node->uuid());
         chand_->subchannel_refcount_map_.erase(it);
       }
     }
@@ -844,8 +880,9 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
       UniquePtr<ConnectivityStateWatcherInterface> watcher) override {
     auto& watcher_wrapper = watcher_map_[watcher.get()];
     GPR_ASSERT(watcher_wrapper == nullptr);
-    watcher_wrapper = New<WatcherWrapper>(
-        std::move(watcher), Ref(DEBUG_LOCATION, "WatcherWrapper"));
+    watcher_wrapper = New<WatcherWrapper>(std::move(watcher),
+                                          Ref(DEBUG_LOCATION, "WatcherWrapper"),
+                                          initial_state);
     subchannel_->WatchConnectivityState(
         initial_state,
         UniquePtr<char>(gpr_strdup(health_check_service_name_.get())),
@@ -870,12 +907,46 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
     return subchannel_->channel_args();
   }
 
+  void UpdateHealthCheckServiceName(UniquePtr<char> health_check_service_name) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+      gpr_log(GPR_INFO,
+              "chand=%p: subchannel wrapper %p: updating health check service "
+              "name from \"%s\" to \"%s\"",
+              chand_, this, health_check_service_name_.get(),
+              health_check_service_name.get());
+    }
+    for (auto& p : watcher_map_) {
+      WatcherWrapper*& watcher_wrapper = p.second;
+      // Cancel the current watcher and create a new one using the new
+      // health check service name.
+      // TODO(roth): If there is not already an existing health watch
+      // call for the new name, then the watcher will initially report
+      // state CONNECTING.  If the LB policy is currently reporting
+      // state READY, this may cause it to switch to CONNECTING before
+      // switching back to READY.  This could cause a small delay for
+      // RPCs being started on the channel.  If/when this becomes a
+      // problem, we may be able to handle it by waiting for the new
+      // watcher to report READY before we use it to replace the old one.
+      WatcherWrapper* replacement = watcher_wrapper->MakeReplacement();
+      subchannel_->CancelConnectivityStateWatch(
+          health_check_service_name_.get(), watcher_wrapper);
+      watcher_wrapper = replacement;
+      subchannel_->WatchConnectivityState(
+          replacement->last_seen_state(),
+          UniquePtr<char>(gpr_strdup(health_check_service_name.get())),
+          OrphanablePtr<Subchannel::ConnectivityStateWatcherInterface>(
+              replacement));
+    }
+    // Save the new health check service name.
+    health_check_service_name_ = std::move(health_check_service_name);
+  }
+
   // Caller must be holding the control-plane combiner.
   ConnectedSubchannel* connected_subchannel() const {
     return connected_subchannel_.get();
   }
 
-  // Caller must be holding the data-plane combiner.
+  // Caller must be holding the data-plane mutex.
   ConnectedSubchannel* connected_subchannel_in_data_plane() const {
     return connected_subchannel_in_data_plane_.get();
   }
@@ -904,8 +975,11 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
     WatcherWrapper(
         UniquePtr<SubchannelInterface::ConnectivityStateWatcherInterface>
             watcher,
-        RefCountedPtr<SubchannelWrapper> parent)
-        : watcher_(std::move(watcher)), parent_(std::move(parent)) {}
+        RefCountedPtr<SubchannelWrapper> parent,
+        grpc_connectivity_state initial_state)
+        : watcher_(std::move(watcher)),
+          parent_(std::move(parent)),
+          last_seen_state_(initial_state) {}
 
     ~WatcherWrapper() { parent_.reset(DEBUG_LOCATION, "WatcherWrapper"); }
 
@@ -928,9 +1002,21 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
     }
 
     grpc_pollset_set* interested_parties() override {
-      return watcher_->interested_parties();
+      SubchannelInterface::ConnectivityStateWatcherInterface* watcher =
+          watcher_.get();
+      if (watcher_ == nullptr) watcher = replacement_->watcher_.get();
+      return watcher->interested_parties();
     }
 
+    WatcherWrapper* MakeReplacement() {
+      auto* replacement =
+          New<WatcherWrapper>(std::move(watcher_), parent_, last_seen_state_);
+      replacement_ = replacement;
+      return replacement;
+    }
+
+    grpc_connectivity_state last_seen_state() const { return last_seen_state_; }
+
    private:
     class Updater {
      public:
@@ -954,12 +1040,17 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
           gpr_log(GPR_INFO,
                   "chand=%p: processing connectivity change in combiner "
                   "for subchannel wrapper %p subchannel %p "
-                  "(connected_subchannel=%p state=%s)",
+                  "(connected_subchannel=%p state=%s): watcher=%p",
                   self->parent_->parent_->chand_, self->parent_->parent_.get(),
                   self->parent_->parent_->subchannel_,
                   self->connected_subchannel_.get(),
-                  grpc_connectivity_state_name(self->state_));
+                  grpc_connectivity_state_name(self->state_),
+                  self->parent_->watcher_.get());
         }
+        // Ignore update if the parent WatcherWrapper has been replaced
+        // since this callback was scheduled.
+        if (self->parent_->watcher_ == nullptr) return;
+        self->parent_->last_seen_state_ = self->state_;
         self->parent_->parent_->MaybeUpdateConnectedSubchannel(
             std::move(self->connected_subchannel_));
         self->parent_->watcher_->OnConnectivityStateChange(self->state_);
@@ -974,6 +1065,8 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
 
     UniquePtr<SubchannelInterface::ConnectivityStateWatcherInterface> watcher_;
     RefCountedPtr<SubchannelWrapper> parent_;
+    grpc_connectivity_state last_seen_state_;
+    WatcherWrapper* replacement_ = nullptr;
   };
 
   void MaybeUpdateConnectedSubchannel(
@@ -981,7 +1074,7 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
     // Update the connected subchannel only if the channel is not shutting
     // down.  This is because once the channel is shutting down, we
     // ignore picker updates from the LB policy, which means that
-    // ConnectivityStateAndPickerSetter will never process the entries
+    // UpdateStateAndPickerLocked() will never process the entries
     // in chand_->pending_subchannel_updates_.  So we don't want to add
     // entries there that will never be processed, since that would
     // leave dangling refs to the channel and prevent its destruction.
@@ -991,7 +1084,7 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
     if (connected_subchannel_ != connected_subchannel) {
       connected_subchannel_ = std::move(connected_subchannel);
       // Record the new connected subchannel so that it can be updated
-      // in the data plane combiner the next time the picker is updated.
+      // in the data plane mutex the next time the picker is updated.
       chand_->pending_subchannel_updates_[Ref(
           DEBUG_LOCATION, "ConnectedSubchannelUpdate")] = connected_subchannel_;
     }
@@ -1008,159 +1101,10 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
   Map<ConnectivityStateWatcherInterface*, WatcherWrapper*> watcher_map_;
   // To be accessed only in the control plane combiner.
   RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
-  // To be accessed only in the data plane combiner.
+  // To be accessed only in the data plane mutex.
   RefCountedPtr<ConnectedSubchannel> connected_subchannel_in_data_plane_;
 };
 
-//
-// ChannelData::ConnectivityStateAndPickerSetter
-//
-
-// A fire-and-forget class that sets the channel's connectivity state
-// and then hops into the data plane combiner to update the picker.
-// Must be instantiated while holding the control plane combiner.
-// Deletes itself when done.
-class ChannelData::ConnectivityStateAndPickerSetter {
- public:
-  ConnectivityStateAndPickerSetter(
-      ChannelData* chand, grpc_connectivity_state state, const char* reason,
-      UniquePtr<LoadBalancingPolicy::SubchannelPicker> picker)
-      : chand_(chand), picker_(std::move(picker)) {
-    // Clean the control plane when entering IDLE, while holding control plane
-    // combiner.
-    if (picker_ == nullptr) {
-      chand->health_check_service_name_.reset();
-      chand->saved_service_config_.reset();
-      chand->received_first_resolver_result_ = false;
-    }
-    // Update connectivity state here, while holding control plane combiner.
-    grpc_connectivity_state_set(&chand->state_tracker_, state, reason);
-    if (chand->channelz_node_ != nullptr) {
-      chand->channelz_node_->SetConnectivityState(state);
-      chand->channelz_node_->AddTraceEvent(
-          channelz::ChannelTrace::Severity::Info,
-          grpc_slice_from_static_string(
-              channelz::ChannelNode::GetChannelConnectivityStateChangeString(
-                  state)));
-    }
-    // Grab any pending subchannel updates.
-    pending_subchannel_updates_ =
-        std::move(chand_->pending_subchannel_updates_);
-    // Bounce into the data plane combiner to reset the picker.
-    GRPC_CHANNEL_STACK_REF(chand->owning_stack_,
-                           "ConnectivityStateAndPickerSetter");
-    GRPC_CLOSURE_INIT(&closure_, SetPickerInDataPlane, this,
-                      grpc_combiner_scheduler(chand->data_plane_combiner_));
-    GRPC_CLOSURE_SCHED(&closure_, GRPC_ERROR_NONE);
-  }
-
- private:
-  static void SetPickerInDataPlane(void* arg, grpc_error* ignored) {
-    auto* self = static_cast<ConnectivityStateAndPickerSetter*>(arg);
-    // Handle subchannel updates.
-    for (auto& p : self->pending_subchannel_updates_) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
-        gpr_log(GPR_INFO,
-                "chand=%p: updating subchannel wrapper %p data plane "
-                "connected_subchannel to %p",
-                self->chand_, p.first.get(), p.second.get());
-      }
-      p.first->set_connected_subchannel_in_data_plane(std::move(p.second));
-    }
-    // Swap out the picker.  We hang on to the old picker so that it can
-    // be deleted in the control-plane combiner, since that's where we need
-    // to unref the subchannel wrappers that are reffed by the picker.
-    self->picker_.swap(self->chand_->picker_);
-    // Clean the data plane if the updated picker is nullptr.
-    if (self->chand_->picker_ == nullptr) {
-      self->chand_->received_service_config_data_ = false;
-      self->chand_->retry_throttle_data_.reset();
-      self->chand_->service_config_.reset();
-    }
-    // Re-process queued picks.
-    for (QueuedPick* pick = self->chand_->queued_picks_; pick != nullptr;
-         pick = pick->next) {
-      CallData::StartPickLocked(pick->elem, GRPC_ERROR_NONE);
-    }
-    // Pop back into the control plane combiner to delete ourself, so
-    // that we make sure to unref subchannel wrappers there.  This
-    // includes both the ones reffed by the old picker (now stored in
-    // self->picker_) and the ones in self->pending_subchannel_updates_.
-    GRPC_CLOSURE_INIT(&self->closure_, CleanUpInControlPlane, self,
-                      grpc_combiner_scheduler(self->chand_->combiner_));
-    GRPC_CLOSURE_SCHED(&self->closure_, GRPC_ERROR_NONE);
-  }
-
-  static void CleanUpInControlPlane(void* arg, grpc_error* ignored) {
-    auto* self = static_cast<ConnectivityStateAndPickerSetter*>(arg);
-    GRPC_CHANNEL_STACK_UNREF(self->chand_->owning_stack_,
-                             "ConnectivityStateAndPickerSetter");
-    Delete(self);
-  }
-
-  ChannelData* chand_;
-  UniquePtr<LoadBalancingPolicy::SubchannelPicker> picker_;
-  Map<RefCountedPtr<SubchannelWrapper>, RefCountedPtr<ConnectedSubchannel>,
-      RefCountedPtrLess<SubchannelWrapper>>
-      pending_subchannel_updates_;
-  grpc_closure closure_;
-};
-
-//
-// ChannelData::ServiceConfigSetter
-//
-
-// A fire-and-forget class that sets the channel's service config data
-// in the data plane combiner.  Deletes itself when done.
-class ChannelData::ServiceConfigSetter {
- public:
-  ServiceConfigSetter(
-      ChannelData* chand,
-      Optional<internal::ClientChannelGlobalParsedConfig::RetryThrottling>
-          retry_throttle_data,
-      RefCountedPtr<ServiceConfig> service_config)
-      : chand_(chand),
-        retry_throttle_data_(retry_throttle_data),
-        service_config_(std::move(service_config)) {
-    GRPC_CHANNEL_STACK_REF(chand->owning_stack_, "ServiceConfigSetter");
-    GRPC_CLOSURE_INIT(&closure_, SetServiceConfigData, this,
-                      grpc_combiner_scheduler(chand->data_plane_combiner_));
-    GRPC_CLOSURE_SCHED(&closure_, GRPC_ERROR_NONE);
-  }
-
- private:
-  static void SetServiceConfigData(void* arg, grpc_error* ignored) {
-    ServiceConfigSetter* self = static_cast<ServiceConfigSetter*>(arg);
-    ChannelData* chand = self->chand_;
-    // Update channel state.
-    chand->received_service_config_data_ = true;
-    if (self->retry_throttle_data_.has_value()) {
-      chand->retry_throttle_data_ =
-          internal::ServerRetryThrottleMap::GetDataForServer(
-              chand->server_name_.get(),
-              self->retry_throttle_data_.value().max_milli_tokens,
-              self->retry_throttle_data_.value().milli_token_ratio);
-    }
-    chand->service_config_ = std::move(self->service_config_);
-    // Apply service config to queued picks.
-    for (QueuedPick* pick = chand->queued_picks_; pick != nullptr;
-         pick = pick->next) {
-      CallData* calld = static_cast<CallData*>(pick->elem->call_data);
-      calld->MaybeApplyServiceConfigToCallLocked(pick->elem);
-    }
-    // Clean up.
-    GRPC_CHANNEL_STACK_UNREF(self->chand_->owning_stack_,
-                             "ServiceConfigSetter");
-    Delete(self);
-  }
-
-  ChannelData* chand_;
-  Optional<internal::ClientChannelGlobalParsedConfig::RetryThrottling>
-      retry_throttle_data_;
-  RefCountedPtr<ServiceConfig> service_config_;
-  grpc_closure closure_;
-};
-
 //
 // ChannelData::ExternalConnectivityWatcher::WatcherList
 //
@@ -1318,11 +1262,6 @@ class ChannelData::ClientChannelControlHelper
         chand_, subchannel, std::move(health_check_service_name));
   }
 
-  grpc_channel* CreateChannel(const char* target,
-                              const grpc_channel_args& args) override {
-    return chand_->client_channel_factory_->CreateChannel(target, &args);
-  }
-
   void UpdateState(
       grpc_connectivity_state state,
       UniquePtr<LoadBalancingPolicy::SubchannelPicker> picker) override {
@@ -1336,20 +1275,18 @@ class ChannelData::ClientChannelControlHelper
     }
     // Do update only if not shutting down.
     if (disconnect_error == GRPC_ERROR_NONE) {
-      // Will delete itself.
-      New<ConnectivityStateAndPickerSetter>(chand_, state, "helper",
-                                            std::move(picker));
+      chand_->UpdateStateAndPickerLocked(state, "helper", std::move(picker));
     }
   }
 
   // No-op -- we should never get this from ResolvingLoadBalancingPolicy.
   void RequestReresolution() override {}
 
-  void AddTraceEvent(TraceSeverity severity, const char* message) override {
+  void AddTraceEvent(TraceSeverity severity, StringView message) override {
     if (chand_->channelz_node_ != nullptr) {
       chand_->channelz_node_->AddTraceEvent(
           ConvertSeverityEnum(severity),
-          grpc_slice_from_copied_string(message));
+          grpc_slice_from_copied_buffer(message.data(), message.size()));
     }
   }
 
@@ -1422,7 +1359,6 @@ ChannelData::ChannelData(grpc_channel_element_args* args, grpc_error** error)
       client_channel_factory_(
           ClientChannelFactory::GetFromChannelArgs(args->channel_args)),
       channelz_node_(GetChannelzNode(args->channel_args)),
-      data_plane_combiner_(grpc_combiner_create()),
       combiner_(grpc_combiner_create()),
       interested_parties_(grpc_pollset_set_create()),
       subchannel_pool_(GetSubchannelPool(args->channel_args)),
@@ -1495,13 +1431,108 @@ ChannelData::~ChannelData() {
   // Stop backup polling.
   grpc_client_channel_stop_backup_polling(interested_parties_);
   grpc_pollset_set_destroy(interested_parties_);
-  GRPC_COMBINER_UNREF(data_plane_combiner_, "client_channel");
   GRPC_COMBINER_UNREF(combiner_, "client_channel");
   GRPC_ERROR_UNREF(disconnect_error_.Load(MemoryOrder::RELAXED));
   grpc_connectivity_state_destroy(&state_tracker_);
   gpr_mu_destroy(&info_mu_);
 }
 
+void ChannelData::UpdateStateAndPickerLocked(
+    grpc_connectivity_state state, const char* reason,
+    UniquePtr<LoadBalancingPolicy::SubchannelPicker> picker) {
+  // Clean the control plane when entering IDLE.
+  if (picker_ == nullptr) {
+    health_check_service_name_.reset();
+    saved_service_config_.reset();
+    received_first_resolver_result_ = false;
+  }
+  // Update connectivity state.
+  grpc_connectivity_state_set(&state_tracker_, state, reason);
+  if (channelz_node_ != nullptr) {
+    channelz_node_->SetConnectivityState(state);
+    channelz_node_->AddTraceEvent(
+        channelz::ChannelTrace::Severity::Info,
+        grpc_slice_from_static_string(
+            channelz::ChannelNode::GetChannelConnectivityStateChangeString(
+                state)));
+  }
+  // Grab data plane lock to do subchannel updates and update the picker.
+  //
+  // Note that we want to minimize the work done while holding the data
+  // plane lock, to keep the critical section small.  So, for all of the
+  // objects that we might wind up unreffing here, we actually hold onto
+  // the refs until after we release the lock, and then unref them at
+  // that point.  This includes the following:
+  // - refs to subchannel wrappers in the keys of pending_subchannel_updates_
+  // - ref stored in retry_throttle_data_
+  // - ref stored in service_config_
+  // - ownership of the existing picker in picker_
+  RefCountedPtr<ServerRetryThrottleData> retry_throttle_data_to_unref;
+  RefCountedPtr<ServiceConfig> service_config_to_unref;
+  {
+    MutexLock lock(&data_plane_mu_);
+    // Handle subchannel updates.
+    for (auto& p : pending_subchannel_updates_) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+        gpr_log(GPR_INFO,
+                "chand=%p: updating subchannel wrapper %p data plane "
+                "connected_subchannel to %p",
+                this, p.first.get(), p.second.get());
+      }
+      // Note: We do not remove the entry from pending_subchannel_updates_
+      // here, since this would unref the subchannel wrapper; instead,
+      // we wait until we've released the lock to clear the map.
+      p.first->set_connected_subchannel_in_data_plane(std::move(p.second));
+    }
+    // Swap out the picker.
+    // Note: Original value will be destroyed after the lock is released.
+    picker_.swap(picker);
+    // Clean the data plane if the updated picker is nullptr.
+    if (picker_ == nullptr) {
+      received_service_config_data_ = false;
+      // Note: We save the objects to unref until after the lock is released.
+      retry_throttle_data_to_unref = std::move(retry_throttle_data_);
+      service_config_to_unref = std::move(service_config_);
+    }
+    // Re-process queued picks.
+    for (QueuedPick* pick = queued_picks_; pick != nullptr; pick = pick->next) {
+      grpc_call_element* elem = pick->elem;
+      CallData* calld = static_cast<CallData*>(elem->call_data);
+      grpc_error* error = GRPC_ERROR_NONE;
+      if (calld->PickSubchannelLocked(elem, &error)) {
+        calld->AsyncPickDone(elem, error);
+      }
+    }
+  }
+  // Clear the pending update map after releasing the lock, to keep the
+  // critical section small.
+  pending_subchannel_updates_.clear();
+}
+
+void ChannelData::UpdateServiceConfigLocked(
+    RefCountedPtr<ServerRetryThrottleData> retry_throttle_data,
+    RefCountedPtr<ServiceConfig> service_config) {
+  // Grab data plane lock to update service config.
+  //
+  // We defer unreffing the old values (and deallocating memory) until
+  // after releasing the lock to keep the critical section small.
+  {
+    MutexLock lock(&data_plane_mu_);
+    // Update service config.
+    received_service_config_data_ = true;
+    // Old values will be unreffed after lock is released.
+    retry_throttle_data_.swap(retry_throttle_data);
+    service_config_.swap(service_config);
+    // Apply service config to queued picks.
+    for (QueuedPick* pick = queued_picks_; pick != nullptr; pick = pick->next) {
+      CallData* calld = static_cast<CallData*>(pick->elem->call_data);
+      calld->MaybeApplyServiceConfigToCallLocked(pick->elem);
+    }
+  }
+  // Old values will be unreffed after lock is released when they go out
+  // of scope.
+}
+
 void ChannelData::CreateResolvingLoadBalancingPolicyLocked() {
   // Instantiate resolving LB policy.
   LoadBalancingPolicy::Args lb_args;
@@ -1660,6 +1691,11 @@ bool ChannelData::ProcessResolverResultLocked(
     } else {
       chand->health_check_service_name_.reset();
     }
+    // Update health check service name used by existing subchannel wrappers.
+    for (const auto& p : chand->subchannel_wrappers_) {
+      p.first->UpdateHealthCheckServiceName(
+          UniquePtr<char>(gpr_strdup(chand->health_check_service_name_.get())));
+    }
     // Save service config.
     chand->saved_service_config_ = std::move(service_config);
   }
@@ -1668,15 +1704,20 @@ bool ChannelData::ProcessResolverResultLocked(
   // if we feel it is unnecessary.
   if (service_config_changed || !chand->received_first_resolver_result_) {
     chand->received_first_resolver_result_ = true;
-    Optional<internal::ClientChannelGlobalParsedConfig::RetryThrottling>
-        retry_throttle_data;
+    RefCountedPtr<ServerRetryThrottleData> retry_throttle_data;
     if (parsed_service_config != nullptr) {
-      retry_throttle_data = parsed_service_config->retry_throttling();
+      Optional<internal::ClientChannelGlobalParsedConfig::RetryThrottling>
+          retry_throttle_config = parsed_service_config->retry_throttling();
+      if (retry_throttle_config.has_value()) {
+        retry_throttle_data =
+            internal::ServerRetryThrottleMap::GetDataForServer(
+                chand->server_name_.get(),
+                retry_throttle_config.value().max_milli_tokens,
+                retry_throttle_config.value().milli_token_ratio);
+      }
     }
-    // Create service config setter to update channel state in the data
-    // plane combiner.  Destroys itself when done.
-    New<ServiceConfigSetter>(chand, retry_throttle_data,
-                             chand->saved_service_config_);
+    chand->UpdateServiceConfigLocked(std::move(retry_throttle_data),
+                                     chand->saved_service_config_);
   }
   UniquePtr<char> processed_lb_policy_name;
   chand->ProcessLbPolicy(result, parsed_service_config,
@@ -1760,8 +1801,8 @@ void ChannelData::StartTransportOpLocked(void* arg, grpc_error* ignored) {
         static_cast<grpc_connectivity_state>(value) == GRPC_CHANNEL_IDLE) {
       if (chand->disconnect_error() == GRPC_ERROR_NONE) {
         // Enter IDLE state.
-        New<ConnectivityStateAndPickerSetter>(chand, GRPC_CHANNEL_IDLE,
-                                              "channel entering IDLE", nullptr);
+        chand->UpdateStateAndPickerLocked(GRPC_CHANNEL_IDLE,
+                                          "channel entering IDLE", nullptr);
       }
       GRPC_ERROR_UNREF(op->disconnect_with_error);
     } else {
@@ -1770,8 +1811,8 @@ void ChannelData::StartTransportOpLocked(void* arg, grpc_error* ignored) {
                  GRPC_ERROR_NONE);
       chand->disconnect_error_.Store(op->disconnect_with_error,
                                      MemoryOrder::RELEASE);
-      New<ConnectivityStateAndPickerSetter>(
-          chand, GRPC_CHANNEL_SHUTDOWN, "shutdown from API",
+      chand->UpdateStateAndPickerLocked(
+          GRPC_CHANNEL_SHUTDOWN, "shutdown from API",
           UniquePtr<LoadBalancingPolicy::SubchannelPicker>(
               New<LoadBalancingPolicy::TransientFailurePicker>(
                   GRPC_ERROR_REF(op->disconnect_with_error))));
@@ -1932,6 +1973,10 @@ CallData::CallData(grpc_call_element* elem, const ChannelData& chand,
 CallData::~CallData() {
   grpc_slice_unref_internal(path_);
   GRPC_ERROR_UNREF(cancel_error_);
+  if (backend_metric_data_ != nullptr) {
+    backend_metric_data_
+        ->LoadBalancingPolicy::BackendMetricData::~BackendMetricData();
+  }
   // Make sure there are no remaining pending batches.
   for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) {
     GPR_ASSERT(pending_batches_[i].batch == nullptr);
@@ -2010,8 +2055,8 @@ void CallData::StartTransportStreamOpBatch(
   // Add the batch to the pending list.
   calld->PendingBatchesAdd(elem, batch);
   // Check if we've already gotten a subchannel call.
-  // Note that once we have completed the pick, we do not need to enter
-  // the channel combiner, which is more efficient (especially for
+  // Note that once we have picked a subchannel, we do not need to acquire
+  // the channel's data plane mutex, which is more efficient (especially for
   // streaming calls).
   if (calld->subchannel_call_ != nullptr) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
@@ -2023,18 +2068,15 @@ void CallData::StartTransportStreamOpBatch(
     return;
   }
   // We do not yet have a subchannel call.
-  // For batches containing a send_initial_metadata op, enter the channel
-  // combiner to start a pick.
+  // For batches containing a send_initial_metadata op, acquire the
+  // channel's data plane mutex to pick a subchannel.
   if (GPR_LIKELY(batch->send_initial_metadata)) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
-      gpr_log(GPR_INFO, "chand=%p calld=%p: entering client_channel combiner",
+      gpr_log(GPR_INFO,
+              "chand=%p calld=%p: grabbing data plane mutex to perform pick",
               chand, calld);
     }
-    GRPC_CLOSURE_SCHED(
-        GRPC_CLOSURE_INIT(
-            &batch->handler_private.closure, StartPickLocked, elem,
-            grpc_combiner_scheduler(chand->data_plane_combiner())),
-        GRPC_ERROR_NONE);
+    PickSubchannel(elem, GRPC_ERROR_NONE);
   } else {
     // For all other batches, release the call combiner.
     if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
@@ -2160,8 +2202,8 @@ void CallData::RecvTrailingMetadataReadyForLoadBalancingPolicy(
   // Invoke callback to LB policy.
   Metadata trailing_metadata(calld, calld->recv_trailing_metadata_);
   calld->lb_recv_trailing_metadata_ready_(
-      calld->lb_recv_trailing_metadata_ready_user_data_, &trailing_metadata,
-      &calld->lb_call_state_);
+      calld->lb_recv_trailing_metadata_ready_user_data_, error,
+      &trailing_metadata, &calld->lb_call_state_);
   // Chain to original callback.
   GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata_ready_,
                    GRPC_ERROR_REF(error));
@@ -2462,8 +2504,8 @@ void CallData::DoRetry(grpc_call_element* elem,
             this, next_attempt_time - ExecCtx::Get()->Now());
   }
   // Schedule retry after computed delay.
-  GRPC_CLOSURE_INIT(&pick_closure_, StartPickLocked, elem,
-                    grpc_combiner_scheduler(chand->data_plane_combiner()));
+  GRPC_CLOSURE_INIT(&pick_closure_, PickSubchannel, elem,
+                    grpc_schedule_on_exec_ctx);
   grpc_timer_init(&retry_timer_, next_attempt_time, &pick_closure_);
   // Update bookkeeping.
   if (retry_state != nullptr) retry_state->retry_dispatched = true;
@@ -3168,8 +3210,8 @@ void CallData::AddRetriableSendInitialMetadataOp(
     SubchannelCallRetryState* retry_state,
     SubchannelCallBatchData* batch_data) {
   // Maps the number of retries to the corresponding metadata value slice.
-  static const grpc_slice* retry_count_strings[] = {
-      &GRPC_MDSTR_1, &GRPC_MDSTR_2, &GRPC_MDSTR_3, &GRPC_MDSTR_4};
+  const grpc_slice* retry_count_strings[] = {&GRPC_MDSTR_1, &GRPC_MDSTR_2,
+                                             &GRPC_MDSTR_3, &GRPC_MDSTR_4};
   // We need to make a copy of the metadata batch for each attempt, since
   // the filters in the subchannel stack may modify this batch, and we don't
   // want those modifications to be passed forward to subsequent attempts.
@@ -3186,8 +3228,7 @@ void CallData::AddRetriableSendInitialMetadataOp(
   if (GPR_UNLIKELY(retry_state->send_initial_metadata.idx.named
                        .grpc_previous_rpc_attempts != nullptr)) {
     grpc_metadata_batch_remove(&retry_state->send_initial_metadata,
-                               retry_state->send_initial_metadata.idx.named
-                                   .grpc_previous_rpc_attempts);
+                               GRPC_BATCH_GRPC_PREVIOUS_RPC_ATTEMPTS);
   }
   if (GPR_UNLIKELY(num_attempts_completed_ > 0)) {
     grpc_mdelem retry_md = grpc_mdelem_create(
@@ -3197,7 +3238,7 @@ void CallData::AddRetriableSendInitialMetadataOp(
         &retry_state->send_initial_metadata,
         &retry_state
              ->send_initial_metadata_storage[send_initial_metadata_.list.count],
-        retry_md);
+        retry_md, GRPC_BATCH_GRPC_PREVIOUS_RPC_ATTEMPTS);
     if (GPR_UNLIKELY(error != GRPC_ERROR_NONE)) {
       gpr_log(GPR_ERROR, "error adding retry metadata: %s",
               grpc_error_string(error));
@@ -3579,6 +3620,11 @@ void CallData::CreateSubchannelCall(grpc_call_element* elem) {
   }
 }
 
+void CallData::AsyncPickDone(grpc_call_element* elem, grpc_error* error) {
+  GRPC_CLOSURE_INIT(&pick_closure_, PickDone, elem, grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_SCHED(&pick_closure_, error);
+}
+
 void CallData::PickDone(void* arg, grpc_error* error) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
@@ -3601,10 +3647,9 @@ class CallData::QueuedPickCanceller {
  public:
   explicit QueuedPickCanceller(grpc_call_element* elem) : elem_(elem) {
     auto* calld = static_cast<CallData*>(elem->call_data);
-    auto* chand = static_cast<ChannelData*>(elem->channel_data);
     GRPC_CALL_STACK_REF(calld->owning_call_, "QueuedPickCanceller");
     GRPC_CLOSURE_INIT(&closure_, &CancelLocked, this,
-                      grpc_combiner_scheduler(chand->data_plane_combiner()));
+                      grpc_schedule_on_exec_ctx);
     calld->call_combiner_->SetNotifyOnCancel(&closure_);
   }
 
@@ -3613,6 +3658,7 @@ class CallData::QueuedPickCanceller {
     auto* self = static_cast<QueuedPickCanceller*>(arg);
     auto* chand = static_cast<ChannelData*>(self->elem_->channel_data);
     auto* calld = static_cast<CallData*>(self->elem_->call_data);
+    MutexLock lock(chand->data_plane_mu());
     if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: cancelling queued pick: "
@@ -3684,7 +3730,7 @@ void CallData::ApplyServiceConfigToCallLocked(grpc_call_element* elem) {
     // from the client API, reset the deadline timer.
     if (chand->deadline_checking_enabled() && method_params_->timeout() != 0) {
       const grpc_millis per_method_deadline =
-          grpc_timespec_to_millis_round_up(call_start_time_) +
+          grpc_cycle_counter_to_millis_round_up(call_start_time_) +
           method_params_->timeout();
       if (per_method_deadline < deadline_) {
         deadline_ = per_method_deadline;
@@ -3731,29 +3777,44 @@ const char* PickResultTypeName(
       return "COMPLETE";
     case LoadBalancingPolicy::PickResult::PICK_QUEUE:
       return "QUEUE";
-    case LoadBalancingPolicy::PickResult::PICK_TRANSIENT_FAILURE:
-      return "TRANSIENT_FAILURE";
+    case LoadBalancingPolicy::PickResult::PICK_FAILED:
+      return "FAILED";
   }
   GPR_UNREACHABLE_CODE(return "UNKNOWN");
 }
 
-void CallData::StartPickLocked(void* arg, grpc_error* error) {
+void CallData::PickSubchannel(void* arg, grpc_error* error) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
   CallData* calld = static_cast<CallData*>(elem->call_data);
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
-  GPR_ASSERT(calld->connected_subchannel_ == nullptr);
-  GPR_ASSERT(calld->subchannel_call_ == nullptr);
-  // picker's being null means the channel is currently in IDLE state. The
-  // incoming call will make the channel exit IDLE and queue itself.
+  bool pick_complete;
+  {
+    MutexLock lock(chand->data_plane_mu());
+    pick_complete = calld->PickSubchannelLocked(elem, &error);
+  }
+  if (pick_complete) {
+    PickDone(elem, error);
+    GRPC_ERROR_UNREF(error);
+  }
+}
+
+bool CallData::PickSubchannelLocked(grpc_call_element* elem,
+                                    grpc_error** error) {
+  ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
+  GPR_ASSERT(connected_subchannel_ == nullptr);
+  GPR_ASSERT(subchannel_call_ == nullptr);
+  // The picker being null means that the channel is currently in IDLE state.
+  // The incoming call will make the channel exit IDLE.
   if (chand->picker() == nullptr) {
-    // We are currently in the data plane.
-    // Bounce into the control plane to exit IDLE.
-    chand->CheckConnectivityState(true);
-    calld->AddCallToQueuedPicksLocked(elem);
-    return;
+    // Bounce into the control plane combiner to exit IDLE.
+    chand->CheckConnectivityState(/*try_to_connect=*/true);
+    // Queue the pick, so that it will be attempted once the channel
+    // becomes connected.
+    AddCallToQueuedPicksLocked(elem);
+    return false;
   }
   // Apply service config to call if needed.
-  calld->MaybeApplyServiceConfigToCallLocked(elem);
+  MaybeApplyServiceConfigToCallLocked(elem);
   // If this is a retry, use the send_initial_metadata payload that
   // we've cached; otherwise, use the pending batch.  The
   // send_initial_metadata batch will be the first pending batch in the
@@ -3765,42 +3826,38 @@ void CallData::StartPickLocked(void* arg, grpc_error* error) {
   // subchannel's copy of the metadata batch (which is copied for each
   // attempt) to the LB policy instead the one from the parent channel.
   LoadBalancingPolicy::PickArgs pick_args;
-  pick_args.call_state = &calld->lb_call_state_;
+  pick_args.call_state = &lb_call_state_;
   Metadata initial_metadata(
-      calld,
-      calld->seen_send_initial_metadata_
-          ? &calld->send_initial_metadata_
-          : calld->pending_batches_[0]
+      this,
+      seen_send_initial_metadata_
+          ? &send_initial_metadata_
+          : pending_batches_[0]
                 .batch->payload->send_initial_metadata.send_initial_metadata);
   pick_args.initial_metadata = &initial_metadata;
   // Grab initial metadata flags so that we can check later if the call has
   // wait_for_ready enabled.
   const uint32_t send_initial_metadata_flags =
-      calld->seen_send_initial_metadata_
-          ? calld->send_initial_metadata_flags_
-          : calld->pending_batches_[0]
-                .batch->payload->send_initial_metadata
-                .send_initial_metadata_flags;
-  // When done, we schedule this closure to leave the data plane combiner.
-  GRPC_CLOSURE_INIT(&calld->pick_closure_, PickDone, elem,
-                    grpc_schedule_on_exec_ctx);
+      seen_send_initial_metadata_ ? send_initial_metadata_flags_
+                                  : pending_batches_[0]
+                                        .batch->payload->send_initial_metadata
+                                        .send_initial_metadata_flags;
   // Attempt pick.
   auto result = chand->picker()->Pick(pick_args);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: LB pick returned %s (subchannel=%p, error=%s)",
-            chand, calld, PickResultTypeName(result.type),
+            chand, this, PickResultTypeName(result.type),
             result.subchannel.get(), grpc_error_string(result.error));
   }
   switch (result.type) {
-    case LoadBalancingPolicy::PickResult::PICK_TRANSIENT_FAILURE: {
+    case LoadBalancingPolicy::PickResult::PICK_FAILED: {
       // If we're shutting down, fail all RPCs.
       grpc_error* disconnect_error = chand->disconnect_error();
       if (disconnect_error != GRPC_ERROR_NONE) {
         GRPC_ERROR_UNREF(result.error);
-        GRPC_CLOSURE_SCHED(&calld->pick_closure_,
-                           GRPC_ERROR_REF(disconnect_error));
-        break;
+        if (pick_queued_) RemoveCallFromQueuedPicksLocked(elem);
+        *error = GRPC_ERROR_REF(disconnect_error);
+        return true;
       }
       // If wait_for_ready is false, then the error indicates the RPC
       // attempt's final status.
@@ -3808,19 +3865,20 @@ void CallData::StartPickLocked(void* arg, grpc_error* error) {
            GRPC_INITIAL_METADATA_WAIT_FOR_READY) == 0) {
         // Retry if appropriate; otherwise, fail.
         grpc_status_code status = GRPC_STATUS_OK;
-        grpc_error_get_status(result.error, calld->deadline_, &status, nullptr,
+        grpc_error_get_status(result.error, deadline_, &status, nullptr,
                               nullptr, nullptr);
-        if (!calld->enable_retries_ ||
-            !calld->MaybeRetry(elem, nullptr /* batch_data */, status,
-                               nullptr /* server_pushback_md */)) {
+        const bool retried = enable_retries_ &&
+                             MaybeRetry(elem, nullptr /* batch_data */, status,
+                                        nullptr /* server_pushback_md */);
+        if (!retried) {
           grpc_error* new_error =
               GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                   "Failed to pick subchannel", &result.error, 1);
           GRPC_ERROR_UNREF(result.error);
-          GRPC_CLOSURE_SCHED(&calld->pick_closure_, new_error);
+          *error = new_error;
         }
-        if (calld->pick_queued_) calld->RemoveCallFromQueuedPicksLocked(elem);
-        break;
+        if (pick_queued_) RemoveCallFromQueuedPicksLocked(elem);
+        return !retried;
       }
       // If wait_for_ready is true, then queue to retry when we get a new
       // picker.
@@ -3828,26 +3886,26 @@ void CallData::StartPickLocked(void* arg, grpc_error* error) {
     }
     // Fallthrough
     case LoadBalancingPolicy::PickResult::PICK_QUEUE:
-      if (!calld->pick_queued_) calld->AddCallToQueuedPicksLocked(elem);
-      break;
+      if (!pick_queued_) AddCallToQueuedPicksLocked(elem);
+      return false;
     default:  // PICK_COMPLETE
+      if (pick_queued_) RemoveCallFromQueuedPicksLocked(elem);
       // Handle drops.
       if (GPR_UNLIKELY(result.subchannel == nullptr)) {
         result.error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
             "Call dropped by load balancing policy");
       } else {
         // Grab a ref to the connected subchannel while we're still
-        // holding the data plane combiner.
-        calld->connected_subchannel_ =
+        // holding the data plane mutex.
+        connected_subchannel_ =
             chand->GetConnectedSubchannelInDataPlane(result.subchannel.get());
-        GPR_ASSERT(calld->connected_subchannel_ != nullptr);
+        GPR_ASSERT(connected_subchannel_ != nullptr);
       }
-      calld->lb_recv_trailing_metadata_ready_ =
-          result.recv_trailing_metadata_ready;
-      calld->lb_recv_trailing_metadata_ready_user_data_ =
+      lb_recv_trailing_metadata_ready_ = result.recv_trailing_metadata_ready;
+      lb_recv_trailing_metadata_ready_user_data_ =
           result.recv_trailing_metadata_ready_user_data;
-      GRPC_CLOSURE_SCHED(&calld->pick_closure_, result.error);
-      if (calld->pick_queued_) calld->RemoveCallFromQueuedPicksLocked(elem);
+      *error = result.error;
+      return true;
   }
 }