grpc 1.30.1 → 1.32.0.pre1

data/include/grpc/grpc_security.h

@@ -119,6 +119,18 @@ GRPCAPI void grpc_ssl_session_cache_destroy(grpc_ssl_session_cache* cache);
 GRPCAPI grpc_arg
 grpc_ssl_session_cache_create_channel_arg(grpc_ssl_session_cache* cache);
 
+/** --- grpc_call_credentials object.
+
+   A call credentials object represents a way to authenticate on a particular
+   call. These credentials can be composed with a channel credentials object
+   so that they are sent with every call on this channel.  */
+
+typedef struct grpc_call_credentials grpc_call_credentials;
+
+/** Releases a call credentials object.
+   The creator of the credentials object is responsible for its release. */
+GRPCAPI void grpc_call_credentials_release(grpc_call_credentials* creds);
+
 /** --- grpc_channel_credentials object. ---
 
    A channel credentials object represents a way to authenticate a client on a
@@ -133,8 +145,23 @@ GRPCAPI void grpc_channel_credentials_release(grpc_channel_credentials* creds);
 /** Creates default credentials to connect to a google gRPC service.
    WARNING: Do NOT use this credentials to connect to a non-google service as
    this could result in an oauth2 token leak. The security level of the
-   resulting connection is GRPC_PRIVACY_AND_INTEGRITY. */
-GRPCAPI grpc_channel_credentials* grpc_google_default_credentials_create(void);
+   resulting connection is GRPC_PRIVACY_AND_INTEGRITY.
+
+   If specified, the supplied call credentials object will be attached to the
+   returned channel credentials object. The call_credentials object must remain
+   valid throughout the lifetime of the returned grpc_channel_credentials
+   object. It is expected that the call credentials object was generated
+   according to the Application Default Credentials mechanism and asserts the
+   identity of the default service account of the machine. Supplying any other
+   sort of call credential will result in undefined behavior, up to and
+   including the sudden and unexpected failure of RPCs.
+
+   If nullptr is supplied, the returned channel credentials object will use a
+   call credentials object based on the Application Default Credentials
+   mechanism.
+*/
+GRPCAPI grpc_channel_credentials* grpc_google_default_credentials_create(
+    grpc_call_credentials* call_credentials);
 
 /** Callback for getting the SSL roots override from the application.
    In case of success, *pem_roots_certs must be set to a NULL terminated string
@@ -272,24 +299,14 @@ GRPCAPI grpc_channel_credentials* grpc_ssl_credentials_create_ex(
     const char* pem_root_certs, grpc_ssl_pem_key_cert_pair* pem_key_cert_pair,
     const grpc_ssl_verify_peer_options* verify_options, void* reserved);
 
-/** --- grpc_call_credentials object.
-
-   A call credentials object represents a way to authenticate on a particular
-   call. These credentials can be composed with a channel credentials object
-   so that they are sent with every call on this channel.  */
-
-typedef struct grpc_call_credentials grpc_call_credentials;
-
-/** Releases a call credentials object.
-   The creator of the credentials object is responsible for its release. */
-GRPCAPI void grpc_call_credentials_release(grpc_call_credentials* creds);
-
 /** Creates a composite channel credentials object. The security level of
  * resulting connection is determined by channel_creds. */
 GRPCAPI grpc_channel_credentials* grpc_composite_channel_credentials_create(
     grpc_channel_credentials* channel_creds, grpc_call_credentials* call_creds,
     void* reserved);
 
+/** --- composite credentials. */
+
 /** Creates a composite call credentials object. */
 GRPCAPI grpc_call_credentials* grpc_composite_call_credentials_create(
     grpc_call_credentials* creds1, grpc_call_credentials* creds2,

data/include/grpc/grpc_security_constants.h

@@ -139,6 +139,9 @@ typedef enum {
  */
 typedef enum { UDS = 0, LOCAL_TCP } grpc_local_connect_type;
 
+/** The TLS versions that are supported by the SSL stack. **/
+typedef enum { TLS1_2, TLS1_3 } grpc_tls_version;
+
 #ifdef __cplusplus
 }
 #endif

data/include/grpc/impl/codegen/README.md

@@ -0,0 +1,22 @@
+# Welcome to `include/grpc/impl/codegen`
+
+## Why is this directory here?
+
+This directory exists so that generated C++ code can include selected files upon
+which it depends without having to depend on the entire gRPC C++ library. This
+directory thus exists to support `include/grpcpp/impl/codegen`. This constraint
+is particularly relevant for users of bazel, particularly if they use the
+multi-lingual `proto_library` target type. Generated code that uses this target
+only depends on the gRPC C++ targets associated with these header files, not the
+entire gRPC C++ codebase since that would make the build time of these types of
+targets excessively large (particularly when they are not even C++ specific).
+
+## What should user code do?
+
+User code should *not* include anything from this directory. Only generated code
+and gRPC library code should include contents from this directory. C++ user code
+should instead include contents from the main `grpcpp` directory or its
+accessible subcomponents like `grpcpp/support`. It is possible that we may
+remove this directory altogether if the motivations for its existence are no
+longer strong enough (e.g., if the gRPC C++ library no longer has a need for an
+`impl/codegen` directory of its own).

data/include/grpc/impl/codegen/grpc_types.h

@@ -203,18 +203,18 @@ typedef struct {
 /** Should BDP probing be performed? */
 #define GRPC_ARG_HTTP2_BDP_PROBE "grpc.http2.bdp_probe"
 /** Minimum time between sending successive ping frames without receiving any
-    data/header/window_update frame, Int valued, milliseconds. */
+    data/header frame, Int valued, milliseconds. */
 #define GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS \
   "grpc.http2.min_time_between_pings_ms"
 /** Minimum allowed time between a server receiving successive ping frames
-   without sending any data/header/window_update frame. Int valued, milliseconds
+   without sending any data/header frame. Int valued, milliseconds
  */
 #define GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS \
   "grpc.http2.min_ping_interval_without_data_ms"
 /** Channel arg to override the http2 :scheme header */
 #define GRPC_ARG_HTTP2_SCHEME "grpc.http2_scheme"
 /** How many pings can we send before needing to send a
-   data/header/window_update frame? (0 indicates that an infinite number of
+   data/header frame? (0 indicates that an infinite number of
    pings can be sent without sending a data frame or header frame) */
 #define GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA \
   "grpc.http2.max_pings_without_data"
@@ -674,8 +674,10 @@ typedef struct grpc_op {
       const char** error_string;
     } recv_status_on_client;
     struct grpc_op_recv_close_on_server {
-      /** out argument, set to 1 if the call failed in any way (seen as a
-          cancellation on the server), or 0 if the call succeeded */
+      /** out argument, set to 1 if the call failed at the server for
+          a reason other than a non-OK status (cancel, deadline
+          exceeded, network failure, etc.), 0 otherwise (RPC processing ran to
+          completion and was able to provide any status from the server) */
       int* cancelled;
     } recv_close_on_server;
   } data;

data/include/grpc/impl/codegen/port_platform.h

@@ -27,19 +27,17 @@
  *  - some syscalls to be made directly
  */
 
-/*
- * Defines GRPC_USE_ABSL to use Abseil Common Libraries (C++)
- */
-#ifndef GRPC_USE_ABSL
-#define GRPC_USE_ABSL 1
-#endif
-
 /*
  * Defines GPR_ABSEIL_SYNC to use synchronization features from Abseil
  */
 #ifndef GPR_ABSEIL_SYNC
-/* #define GPR_ABSEIL_SYNC 1 */
+#if defined(__APPLE__)
+// This is disabled on Apple platforms because macos/grpc_basictests_c_cpp
+// fails with this. https://github.com/grpc/grpc/issues/23661
+#else
+#define GPR_ABSEIL_SYNC 1
 #endif
+#endif  // GPR_ABSEIL_SYNC
 
 /* Get windows.h included everywhere (we need it) */
 #if defined(_WIN64) || defined(WIN64) || defined(_WIN32) || defined(WIN32)
@@ -112,31 +110,6 @@
 #define GPR_WINDOWS_ATOMIC 1
 #define GPR_MSVC_TLS 1
 #endif
-#elif defined(GPR_MANYLINUX1)
-// TODO(atash): manylinux1 is just another __linux__ but with ancient
-// libraries; it should be integrated with the `__linux__` definitions below.
-#define GPR_PLATFORM_STRING "manylinux"
-#define GPR_POSIX_CRASH_HANDLER 1
-#define GPR_CPU_POSIX 1
-#define GPR_GCC_ATOMIC 1
-#define GPR_GCC_TLS 1
-#define GPR_LINUX 1
-#define GPR_LINUX_LOG 1
-#define GPR_SUPPORT_CHANNELS_FROM_FD 1
-#define GPR_LINUX_ENV 1
-#define GPR_POSIX_TMPFILE 1
-#define GPR_POSIX_STRING 1
-#define GPR_POSIX_SUBPROCESS 1
-#define GPR_POSIX_SYNC 1
-#define GPR_POSIX_TIME 1
-#define GPR_HAS_PTHREAD_H 1
-#define GPR_GETPID_IN_UNISTD_H 1
-#ifdef _LP64
-#define GPR_ARCH_64 1
-#else /* _LP64 */
-#define GPR_ARCH_32 1
-#endif /* _LP64 */
-#include <linux/version.h>
 #elif defined(ANDROID) || defined(__ANDROID__)
 #define GPR_PLATFORM_STRING "android"
 #define GPR_ANDROID 1

data/src/core/ext/filters/client_channel/backend_metric.cc

@@ -21,6 +21,7 @@
 #include "absl/strings/string_view.h"
 
 #include "udpa/data/orca/v1/orca_load_report.upb.h"
+#include "upb/upb.hpp"
 
 #include "src/core/lib/gprpp/map.h"
 
@@ -31,17 +32,19 @@ namespace {
 template <typename EntryType>
 std::map<absl::string_view, double, StringLess> ParseMap(
     udpa_data_orca_v1_OrcaLoadReport* msg,
-    EntryType** (*entry_func)(udpa_data_orca_v1_OrcaLoadReport*, size_t*),
+    const EntryType* (*entry_func)(const udpa_data_orca_v1_OrcaLoadReport*,
+                                   size_t*),
     upb_strview (*key_func)(const EntryType*),
     double (*value_func)(const EntryType*), Arena* arena) {
   std::map<absl::string_view, double, StringLess> result;
-  size_t size;
-  const auto* const* entries = entry_func(msg, &size);
-  for (size_t i = 0; i < size; ++i) {
-    upb_strview key_view = key_func(entries[i]);
-    char* key = static_cast<char*>(arena->Alloc(key_view.size + 1));
+  size_t i = UPB_MAP_BEGIN;
+  while (true) {
+    const auto* entry = entry_func(msg, &i);
+    if (entry == nullptr) break;
+    upb_strview key_view = key_func(entry);
+    char* key = static_cast<char*>(arena->Alloc(key_view.size));
     memcpy(key, key_view.data, key_view.size);
-    result[absl::string_view(key, key_view.size)] = value_func(entries[i]);
+    result[absl::string_view(key, key_view.size)] = value_func(entry);
   }
   return result;
 }
@@ -67,12 +70,12 @@ const LoadBalancingPolicy::BackendMetricData* ParseBackendMetricData(
       udpa_data_orca_v1_OrcaLoadReport_rps(msg);
   backend_metric_data->request_cost =
       ParseMap<udpa_data_orca_v1_OrcaLoadReport_RequestCostEntry>(
-          msg, udpa_data_orca_v1_OrcaLoadReport_mutable_request_cost,
+          msg, udpa_data_orca_v1_OrcaLoadReport_request_cost_next,
           udpa_data_orca_v1_OrcaLoadReport_RequestCostEntry_key,
           udpa_data_orca_v1_OrcaLoadReport_RequestCostEntry_value, arena);
   backend_metric_data->utilization =
       ParseMap<udpa_data_orca_v1_OrcaLoadReport_UtilizationEntry>(
-          msg, udpa_data_orca_v1_OrcaLoadReport_mutable_utilization,
+          msg, udpa_data_orca_v1_OrcaLoadReport_utilization_next,
           udpa_data_orca_v1_OrcaLoadReport_UtilizationEntry_key,
           udpa_data_orca_v1_OrcaLoadReport_UtilizationEntry_value, arena);
   return backend_metric_data;

data/src/core/ext/filters/client_channel/backup_poller.cc

@@ -106,6 +106,7 @@ static void g_poller_unref() {
                                       grpc_schedule_on_exec_ctx));
     gpr_mu_unlock(p->pollset_mu);
     grpc_timer_cancel(&p->polling_timer);
+    backup_poller_shutdown_unref(p);
   } else {
     gpr_mu_unlock(&g_poller_mu);
   }
@@ -143,8 +144,8 @@ static void g_poller_init_locked() {
     g_poller->shutting_down = false;
     grpc_pollset_init(g_poller->pollset, &g_poller->pollset_mu);
     gpr_ref_init(&g_poller->refs, 0);
-    // one for timer cancellation, one for pollset shutdown
-    gpr_ref_init(&g_poller->shutdown_refs, 2);
+    // one for timer cancellation, one for pollset shutdown, one for g_poller
+    gpr_ref_init(&g_poller->shutdown_refs, 3);
     GRPC_CLOSURE_INIT(&g_poller->run_poller_closure, run_poller, g_poller,
                       grpc_schedule_on_exec_ctx);
     grpc_timer_init(&g_poller->polling_timer,

data/src/core/ext/filters/client_channel/client_channel.cc

@@ -28,6 +28,8 @@
 
 #include <set>
 
+#include "absl/strings/numbers.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/string_view.h"
 
 #include <grpc/support/alloc.h>
@@ -40,6 +42,7 @@
 
 #include "src/core/ext/filters/client_channel/backend_metric.h"
 #include "src/core/ext/filters/client_channel/backup_poller.h"
+#include "src/core/ext/filters/client_channel/config_selector.h"
 #include "src/core/ext/filters/client_channel/global_subchannel_pool.h"
 #include "src/core/ext/filters/client_channel/http_connect_handshaker.h"
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
@@ -149,12 +152,16 @@ class ChannelData {
   bool received_service_config_data() const {
     return received_service_config_data_;
   }
+  grpc_error* resolver_transient_failure_error() const {
+    return resolver_transient_failure_error_;
+  }
   RefCountedPtr<ServerRetryThrottleData> retry_throttle_data() const {
     return retry_throttle_data_;
   }
   RefCountedPtr<ServiceConfig> service_config() const {
     return service_config_;
   }
+  ConfigSelector* config_selector() const { return config_selector_.get(); }
   WorkSerializer* work_serializer() const { return work_serializer_.get(); }
 
   RefCountedPtr<ConnectedSubchannel> GetConnectedSubchannelInDataPlane(
@@ -166,29 +173,14 @@ class ChannelData {
                                       grpc_connectivity_state* state,
                                       grpc_closure* on_complete,
                                       grpc_closure* watcher_timer_init) {
-    auto* watcher = new ExternalConnectivityWatcher(
-        this, pollent, state, on_complete, watcher_timer_init);
-    {
-      MutexLock lock(&external_watchers_mu_);
-      // Will be deleted when the watch is complete.
-      GPR_ASSERT(external_watchers_[on_complete] == nullptr);
-      external_watchers_[on_complete] = watcher;
-    }
-    watcher->Start();
+    new ExternalConnectivityWatcher(this, pollent, state, on_complete,
+                                    watcher_timer_init);
   }
 
   void RemoveExternalConnectivityWatcher(grpc_closure* on_complete,
                                          bool cancel) {
-    ExternalConnectivityWatcher* watcher = nullptr;
-    {
-      MutexLock lock(&external_watchers_mu_);
-      auto it = external_watchers_.find(on_complete);
-      if (it != external_watchers_.end()) {
-        watcher = it->second;
-        external_watchers_.erase(it);
-      }
-    }
-    if (watcher != nullptr && cancel) watcher->Cancel();
+    ExternalConnectivityWatcher::RemoveWatcherFromExternalWatchersMap(
+        this, on_complete, cancel);
   }
 
   int NumExternalConnectivityWatchers() const {
@@ -219,13 +211,19 @@ class ChannelData {
 
     ~ExternalConnectivityWatcher();
 
-    void Start();
+    // Removes the watcher from the external_watchers_ map.
+    static void RemoveWatcherFromExternalWatchersMap(ChannelData* chand,
+                                                     grpc_closure* on_complete,
+                                                     bool cancel);
 
-    void Notify(grpc_connectivity_state state) override;
+    void Notify(grpc_connectivity_state state,
+                const absl::Status& /* status */) override;
 
     void Cancel();
 
    private:
+    // Adds the watcher to state_tracker_. Consumes the ref that is passed to it
+    // from Start().
     void AddWatcherLocked();
     void RemoveWatcherLocked();
 
@@ -238,37 +236,51 @@ class ChannelData {
     Atomic<bool> done_{false};
   };
 
+  class ChannelConfigHelper
+      : public ResolvingLoadBalancingPolicy::ChannelConfigHelper {
+   public:
+    explicit ChannelConfigHelper(ChannelData* chand) : chand_(chand) {}
+
+    ApplyServiceConfigResult ApplyServiceConfig(
+        const Resolver::Result& result) override;
+
+    void ApplyConfigSelector(
+        bool service_config_changed,
+        RefCountedPtr<ConfigSelector> config_selector) override;
+
+    void ResolverTransientFailure(grpc_error* error) override;
+
+   private:
+    static void ProcessLbPolicy(
+        const Resolver::Result& resolver_result,
+        const internal::ClientChannelGlobalParsedConfig* parsed_service_config,
+        RefCountedPtr<LoadBalancingPolicy::Config>* lb_policy_config);
+
+    ChannelData* chand_;
+  };
+
   ChannelData(grpc_channel_element_args* args, grpc_error** error);
   ~ChannelData();
 
   void UpdateStateAndPickerLocked(
-      grpc_connectivity_state state, const char* reason,
+      grpc_connectivity_state state, const absl::Status& status,
+      const char* reason,
       std::unique_ptr<LoadBalancingPolicy::SubchannelPicker> picker);
 
-  void UpdateServiceConfigLocked(
-      RefCountedPtr<ServerRetryThrottleData> retry_throttle_data,
-      RefCountedPtr<ServiceConfig> service_config);
+  void UpdateServiceConfigInDataPlaneLocked(
+      bool service_config_changed,
+      RefCountedPtr<ConfigSelector> config_selector);
 
   void CreateResolvingLoadBalancingPolicyLocked();
 
   void DestroyResolvingLoadBalancingPolicyLocked();
 
-  static bool ProcessResolverResultLocked(
-      void* arg, const Resolver::Result& result,
-      RefCountedPtr<LoadBalancingPolicy::Config>* lb_policy_config,
-      grpc_error** service_config_error, bool* no_valid_service_config);
-
   grpc_error* DoPingLocked(grpc_transport_op* op);
 
   void StartTransportOpLocked(grpc_transport_op* op);
 
   void TryToConnectLocked();
 
-  void ProcessLbPolicy(
-      const Resolver::Result& resolver_result,
-      const internal::ClientChannelGlobalParsedConfig* parsed_service_config,
-      RefCountedPtr<LoadBalancingPolicy::Config>* lb_policy_config);
-
   //
   // Fields set at construction and never modified.
   //
@@ -282,6 +294,7 @@ class ChannelData {
   grpc_core::UniquePtr<char> server_name_;
   grpc_core::UniquePtr<char> target_uri_;
   channelz::ChannelNode* channelz_node_;
+  ChannelConfigHelper channel_config_helper_;
 
   //
   // Fields used in the data plane.  Guarded by data_plane_mu.
@@ -290,9 +303,11 @@ class ChannelData {
   std::unique_ptr<LoadBalancingPolicy::SubchannelPicker> picker_;
   QueuedPick* queued_picks_ = nullptr;  // Linked list of queued picks.
   // Data from service config.
+  grpc_error* resolver_transient_failure_error_ = GRPC_ERROR_NONE;
   bool received_service_config_data_ = false;
   RefCountedPtr<ServerRetryThrottleData> retry_throttle_data_;
   RefCountedPtr<ServiceConfig> service_config_;
+  RefCountedPtr<ConfigSelector> config_selector_;
 
   //
   // Fields used in the control plane.  Guarded by work_serializer.
@@ -304,6 +319,7 @@ class ChannelData {
   ConnectivityStateTracker state_tracker_;
   grpc_core::UniquePtr<char> health_check_service_name_;
   RefCountedPtr<ServiceConfig> saved_service_config_;
+  RefCountedPtr<ConfigSelector> saved_config_selector_;
   bool received_first_resolver_result_ = false;
   // The number of SubchannelWrapper instances referencing a given Subchannel.
   std::map<Subchannel*, int> subchannel_refcount_map_;
@@ -316,6 +332,7 @@ class ChannelData {
   // applied in the data plane mutex when the picker is updated.
   std::map<RefCountedPtr<SubchannelWrapper>, RefCountedPtr<ConnectedSubchannel>>
       pending_subchannel_updates_;
+  int keepalive_time_ = -1;
 
   //
   // Fields accessed from both data plane mutex and control plane
@@ -336,7 +353,8 @@ class ChannelData {
   // synchronously via grpc_channel_num_external_connectivity_watchers().
   //
   mutable Mutex external_watchers_mu_;
-  std::map<grpc_closure*, ExternalConnectivityWatcher*> external_watchers_;
+  std::map<grpc_closure*, RefCountedPtr<ExternalConnectivityWatcher>>
+      external_watchers_;
 };
 
 //
@@ -356,9 +374,6 @@ class CallData {
 
   RefCountedPtr<SubchannelCall> subchannel_call() { return subchannel_call_; }
 
-  // Invoked by channel for queued picks once resolver results are available.
-  void MaybeApplyServiceConfigToCallLocked(grpc_call_element* elem);
-
   // Invoked by channel for queued picks when the picker is updated.
   static void PickSubchannel(void* arg, grpc_error* error);
 
@@ -392,7 +407,8 @@ class CallData {
     iterator begin() const override {
       static_assert(sizeof(grpc_linked_mdelem*) <= sizeof(intptr_t),
                     "iterator size too large");
-      return iterator(this, reinterpret_cast<intptr_t>(batch_->list.head));
+      return iterator(
+          this, reinterpret_cast<intptr_t>(MaybeSkipEntry(batch_->list.head)));
     }
     iterator end() const override {
       static_assert(sizeof(grpc_linked_mdelem*) <= sizeof(intptr_t),
@@ -409,11 +425,19 @@ class CallData {
     }
 
    private:
+    grpc_linked_mdelem* MaybeSkipEntry(grpc_linked_mdelem* entry) const {
+      if (entry != nullptr && batch_->idx.named.path == entry) {
+        return entry->next;
+      }
+      return entry;
+    }
+
     intptr_t IteratorHandleNext(intptr_t handle) const override {
       grpc_linked_mdelem* linked_mdelem =
           reinterpret_cast<grpc_linked_mdelem*>(handle);
-      return reinterpret_cast<intptr_t>(linked_mdelem->next);
+      return reinterpret_cast<intptr_t>(MaybeSkipEntry(linked_mdelem->next));
     }
+
     std::pair<absl::string_view, absl::string_view> IteratorHandleGet(
         intptr_t handle) const override {
       grpc_linked_mdelem* linked_mdelem =
@@ -746,13 +770,17 @@ class CallData {
   void CreateSubchannelCall(grpc_call_element* elem);
   // Invoked when a pick is completed, on both success or failure.
   static void PickDone(void* arg, grpc_error* error);
-  // Removes the call from the channel's list of queued picks.
-  void RemoveCallFromQueuedPicksLocked(grpc_call_element* elem);
-  // Adds the call to the channel's list of queued picks.
-  void AddCallToQueuedPicksLocked(grpc_call_element* elem);
+  // Removes the call from the channel's list of queued picks if present.
+  void MaybeRemoveCallFromQueuedPicksLocked(grpc_call_element* elem);
+  // Adds the call to the channel's list of queued picks if not already present.
+  void MaybeAddCallToQueuedPicksLocked(grpc_call_element* elem);
   // Applies service config to the call.  Must be invoked once we know
   // that the resolver has returned results to the channel.
-  void ApplyServiceConfigToCallLocked(grpc_call_element* elem);
+  // If an error is returned, the error indicates the status with which
+  // the call should be failed.
+  grpc_error* ApplyServiceConfigToCallLocked(
+      grpc_call_element* elem, grpc_metadata_batch* initial_metadata);
+  void MaybeInvokeConfigSelectorCommitCallback();
 
   // State for handling deadlines.
   // The code in deadline_filter.c requires this to be the first field.
@@ -773,6 +801,7 @@ class CallData {
   RefCountedPtr<ServerRetryThrottleData> retry_throttle_data_;
   const ClientChannelMethodParsedConfig* method_params_ = nullptr;
   std::map<const char*, absl::string_view> call_attributes_;
+  std::function<void()> on_call_committed_;
 
   RefCountedPtr<SubchannelCall> subchannel_call_;
 
@@ -945,6 +974,10 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
 
   void ResetBackoff() override { subchannel_->ResetBackoff(); }
 
+  void ThrottleKeepaliveTime(int new_keepalive_time) {
+    subchannel_->ThrottleKeepaliveTime(new_keepalive_time);
+  }
+
   const grpc_channel_args* channel_args() override {
     return subchannel_->channel_args();
   }
@@ -1075,6 +1108,32 @@ class ChannelData::SubchannelWrapper : public SubchannelInterface {
                 watcher_.get());
       }
       ConnectivityStateChange state_change = PopConnectivityStateChange();
+      absl::optional<absl::Cord> keepalive_throttling =
+          state_change.status.GetPayload(grpc_core::kKeepaliveThrottlingKey);
+      if (keepalive_throttling.has_value()) {
+        int new_keepalive_time = -1;
+        if (absl::SimpleAtoi(std::string(keepalive_throttling.value()),
+                             &new_keepalive_time)) {
+          if (new_keepalive_time > parent_->chand_->keepalive_time_) {
+            parent_->chand_->keepalive_time_ = new_keepalive_time;
+            if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+              gpr_log(GPR_INFO, "chand=%p: throttling keepalive time to %d",
+                      parent_->chand_, parent_->chand_->keepalive_time_);
+            }
+            // Propagate the new keepalive time to all subchannels. This is so
+            // that new transports created by any subchannel (and not just the
+            // subchannel that received the GOAWAY), use the new keepalive time.
+            for (auto* subchannel_wrapper :
+                 parent_->chand_->subchannel_wrappers_) {
+              subchannel_wrapper->ThrottleKeepaliveTime(new_keepalive_time);
+            }
+          }
+        } else {
+          gpr_log(GPR_ERROR, "chand=%p: Illegal keepalive throttling value %s",
+                  parent_->chand_,
+                  std::string(keepalive_throttling.value()).c_str());
+        }
+      }
       // Ignore update if the parent WatcherWrapper has been replaced
       // since this callback was scheduled.
       if (watcher_ != nullptr) {
@@ -1145,6 +1204,21 @@ ChannelData::ExternalConnectivityWatcher::ExternalConnectivityWatcher(
   grpc_polling_entity_add_to_pollset_set(&pollent_,
                                          chand_->interested_parties_);
   GRPC_CHANNEL_STACK_REF(chand_->owning_stack_, "ExternalConnectivityWatcher");
+  {
+    MutexLock lock(&chand_->external_watchers_mu_);
+    // Will be deleted when the watch is complete.
+    GPR_ASSERT(chand->external_watchers_[on_complete] == nullptr);
+    // Store a ref to the watcher in the external_watchers_ map.
+    chand->external_watchers_[on_complete] =
+        Ref(DEBUG_LOCATION, "AddWatcherToExternalWatchersMapLocked");
+  }
+  // Pass the ref from creating the object to Start().
+  chand_->work_serializer_->Run(
+      [this]() {
+        // The ref is passed to AddWatcherLocked().
+        AddWatcherLocked();
+      },
+      DEBUG_LOCATION);
 }
 
 ChannelData::ExternalConnectivityWatcher::~ExternalConnectivityWatcher() {
@@ -1154,13 +1228,26 @@ ChannelData::ExternalConnectivityWatcher::~ExternalConnectivityWatcher() {
                            "ExternalConnectivityWatcher");
 }
 
-void ChannelData::ExternalConnectivityWatcher::Start() {
-  chand_->work_serializer_->Run([this]() { AddWatcherLocked(); },
-                                DEBUG_LOCATION);
+void ChannelData::ExternalConnectivityWatcher::
+    RemoveWatcherFromExternalWatchersMap(ChannelData* chand,
+                                         grpc_closure* on_complete,
+                                         bool cancel) {
+  RefCountedPtr<ExternalConnectivityWatcher> watcher;
+  {
+    MutexLock lock(&chand->external_watchers_mu_);
+    auto it = chand->external_watchers_.find(on_complete);
+    if (it != chand->external_watchers_.end()) {
+      watcher = std::move(it->second);
+      chand->external_watchers_.erase(it);
+    }
+  }
+  // watcher->Cancel() will hop into the WorkSerializer, so we have to unlock
+  // the mutex before calling it.
+  if (watcher != nullptr && cancel) watcher->Cancel();
 }
 
 void ChannelData::ExternalConnectivityWatcher::Notify(
-    grpc_connectivity_state state) {
+    grpc_connectivity_state state, const absl::Status& /* status */) {
   bool done = false;
   if (!done_.CompareExchangeStrong(&done, true, MemoryOrder::RELAXED,
                                    MemoryOrder::RELAXED)) {
@@ -1194,7 +1281,7 @@ void ChannelData::ExternalConnectivityWatcher::Cancel() {
 
 void ChannelData::ExternalConnectivityWatcher::AddWatcherLocked() {
   Closure::Run(DEBUG_LOCATION, watcher_timer_init_, GRPC_ERROR_NONE);
-  // Add new watcher.
+  // Add new watcher. Pass the ref of the object from creation to OrphanablePtr.
   chand_->state_tracker_.AddWatcher(
       initial_state_, OrphanablePtr<ConnectivityStateWatcherInterface>(this));
 }
@@ -1295,24 +1382,27 @@ class ChannelData::ClientChannelControlHelper
         chand_->client_channel_factory_->CreateSubchannel(new_args);
     grpc_channel_args_destroy(new_args);
     if (subchannel == nullptr) return nullptr;
+    subchannel->ThrottleKeepaliveTime(chand_->keepalive_time_);
     return MakeRefCounted<SubchannelWrapper>(
         chand_, subchannel, std::move(health_check_service_name));
   }
 
   void UpdateState(
-      grpc_connectivity_state state,
+      grpc_connectivity_state state, const absl::Status& status,
       std::unique_ptr<LoadBalancingPolicy::SubchannelPicker> picker) override {
     grpc_error* disconnect_error = chand_->disconnect_error();
     if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
       const char* extra = disconnect_error == GRPC_ERROR_NONE
                               ? ""
                               : " (ignoring -- channel shutting down)";
-      gpr_log(GPR_INFO, "chand=%p: update: state=%s picker=%p%s", chand_,
-              ConnectivityStateName(state), picker.get(), extra);
+      gpr_log(GPR_INFO, "chand=%p: update: state=%s status=(%s) picker=%p%s",
+              chand_, ConnectivityStateName(state), status.ToString().c_str(),
+              picker.get(), extra);
     }
     // Do update only if not shutting down.
     if (disconnect_error == GRPC_ERROR_NONE) {
-      chand_->UpdateStateAndPickerLocked(state, "helper", std::move(picker));
+      chand_->UpdateStateAndPickerLocked(state, status, "helper",
+                                         std::move(picker));
     }
   }
 
@@ -1339,6 +1429,180 @@ class ChannelData::ClientChannelControlHelper
   ChannelData* chand_;
 };
 
+//
+// ChannelData::ChannelConfigHelper
+//
+
+// Synchronous callback from ResolvingLoadBalancingPolicy to process a
+// resolver result update.
+ChannelData::ChannelConfigHelper::ApplyServiceConfigResult
+ChannelData::ChannelConfigHelper::ApplyServiceConfig(
+    const Resolver::Result& result) {
+  ApplyServiceConfigResult service_config_result;
+  RefCountedPtr<ServiceConfig> service_config;
+  // If resolver did not return a service config or returned an invalid service
+  // config, we need a fallback service config.
+  if (result.service_config_error != GRPC_ERROR_NONE) {
+    // If the service config was invalid, then fallback to the saved service
+    // config. If there is no saved config either, use the default service
+    // config.
+    if (chand_->saved_service_config_ != nullptr) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+        gpr_log(GPR_INFO,
+                "chand=%p: resolver returned invalid service config. "
+                "Continuing to use previous service config.",
+                chand_);
+      }
+      service_config = chand_->saved_service_config_;
+    } else if (chand_->default_service_config_ != nullptr) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+        gpr_log(GPR_INFO,
+                "chand=%p: resolver returned invalid service config. Using "
+                "default service config provided by client API.",
+                chand_);
+      }
+      service_config = chand_->default_service_config_;
+    }
+  } else if (result.service_config == nullptr) {
+    if (chand_->default_service_config_ != nullptr) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+        gpr_log(GPR_INFO,
+                "chand=%p: resolver returned no service config. Using default "
+                "service config provided by client API.",
+                chand_);
+      }
+      service_config = chand_->default_service_config_;
+    }
+  } else {
+    service_config = result.service_config;
+  }
+  service_config_result.service_config_error =
+      GRPC_ERROR_REF(result.service_config_error);
+  if (service_config == nullptr &&
+      result.service_config_error != GRPC_ERROR_NONE) {
+    service_config_result.no_valid_service_config = true;
+    return service_config_result;
+  }
+  // Process service config.
+  grpc_core::UniquePtr<char> service_config_json;
+  const internal::ClientChannelGlobalParsedConfig* parsed_service_config =
+      nullptr;
+  if (service_config != nullptr) {
+    parsed_service_config =
+        static_cast<const internal::ClientChannelGlobalParsedConfig*>(
+            service_config->GetGlobalParsedConfig(
+                internal::ClientChannelServiceConfigParser::ParserIndex()));
+  }
+  // Check if the config has changed.
+  service_config_result.service_config_changed =
+      ((service_config == nullptr) !=
+       (chand_->saved_service_config_ == nullptr)) ||
+      (service_config != nullptr &&
+       service_config->json_string() !=
+           chand_->saved_service_config_->json_string());
+  if (service_config_result.service_config_changed) {
+    service_config_json.reset(gpr_strdup(
+        service_config != nullptr ? service_config->json_string().c_str()
+                                  : ""));
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+      gpr_log(GPR_INFO,
+              "chand=%p: resolver returned updated service config: \"%s\"",
+              chand_, service_config_json.get());
+    }
+    // Save health check service name.
+    if (service_config != nullptr) {
+      chand_->health_check_service_name_.reset(
+          gpr_strdup(parsed_service_config->health_check_service_name()));
+    } else {
+      chand_->health_check_service_name_.reset();
+    }
+    // Update health check service name used by existing subchannel wrappers.
+    for (auto* subchannel_wrapper : chand_->subchannel_wrappers_) {
+      subchannel_wrapper->UpdateHealthCheckServiceName(
+          grpc_core::UniquePtr<char>(
+              gpr_strdup(chand_->health_check_service_name_.get())));
+    }
+    // Save service config.
+    chand_->saved_service_config_ = std::move(service_config);
+  }
+  // Find LB policy config.
+  ProcessLbPolicy(result, parsed_service_config,
+                  &service_config_result.lb_policy_config);
+  grpc_core::UniquePtr<char> lb_policy_name(
+      gpr_strdup((service_config_result.lb_policy_config)->name()));
+  // Swap out the data used by GetChannelInfo().
+  {
+    MutexLock lock(&chand_->info_mu_);
+    chand_->info_lb_policy_name_ = std::move(lb_policy_name);
+    if (service_config_json != nullptr) {
+      chand_->info_service_config_json_ = std::move(service_config_json);
+    }
+  }
+  // Return results.
+  return service_config_result;
+}
+
+void ChannelData::ChannelConfigHelper::ApplyConfigSelector(
+    bool service_config_changed,
+    RefCountedPtr<ConfigSelector> config_selector) {
+  chand_->UpdateServiceConfigInDataPlaneLocked(service_config_changed,
+                                               std::move(config_selector));
+}
+
+void ChannelData::ChannelConfigHelper::ResolverTransientFailure(
+    grpc_error* error) {
+  MutexLock lock(&chand_->data_plane_mu_);
+  GRPC_ERROR_UNREF(chand_->resolver_transient_failure_error_);
+  chand_->resolver_transient_failure_error_ = error;
+}
+
+void ChannelData::ChannelConfigHelper::ProcessLbPolicy(
+    const Resolver::Result& resolver_result,
+    const internal::ClientChannelGlobalParsedConfig* parsed_service_config,
+    RefCountedPtr<LoadBalancingPolicy::Config>* lb_policy_config) {
+  // Prefer the LB policy config found in the service config.
+  if (parsed_service_config != nullptr &&
+      parsed_service_config->parsed_lb_config() != nullptr) {
+    *lb_policy_config = parsed_service_config->parsed_lb_config();
+    return;
+  }
+  // Try the deprecated LB policy name from the service config.
+  // If not, try the setting from channel args.
+  const char* policy_name = nullptr;
+  if (parsed_service_config != nullptr &&
+      !parsed_service_config->parsed_deprecated_lb_policy().empty()) {
+    policy_name = parsed_service_config->parsed_deprecated_lb_policy().c_str();
+  } else {
+    const grpc_arg* channel_arg =
+        grpc_channel_args_find(resolver_result.args, GRPC_ARG_LB_POLICY_NAME);
+    policy_name = grpc_channel_arg_get_string(channel_arg);
+  }
+  // Use pick_first if nothing was specified and we didn't select grpclb
+  // above.
+  if (policy_name == nullptr) policy_name = "pick_first";
+  // Now that we have the policy name, construct an empty config for it.
+  Json config_json = Json::Array{Json::Object{
+      {policy_name, Json::Object{}},
+  }};
+  grpc_error* parse_error = GRPC_ERROR_NONE;
+  *lb_policy_config = LoadBalancingPolicyRegistry::ParseLoadBalancingConfig(
+      config_json, &parse_error);
+  // The policy name came from one of three places:
+  // - The deprecated loadBalancingPolicy field in the service config,
+  //   in which case the code in ClientChannelServiceConfigParser
+  //   already verified that the policy does not require a config.
+  // - One of the hard-coded values here, all of which are known to not
+  //   require a config.
+  // - A channel arg, in which case the application did something that
+  //   is a misuse of our API.
+  // In the first two cases, these assertions will always be true.  In
+  // the last case, this is probably fine for now.
+  // TODO(roth): If the last case becomes a problem, add better error
+  // handling here.
+  GPR_ASSERT(*lb_policy_config != nullptr);
+  GPR_ASSERT(parse_error == GRPC_ERROR_NONE);
+}
+
 //
 // ChannelData implementation
 //
@@ -1397,6 +1661,7 @@ ChannelData::ChannelData(grpc_channel_element_args* args, grpc_error** error)
       client_channel_factory_(
           ClientChannelFactory::GetFromChannelArgs(args->channel_args)),
       channelz_node_(GetChannelzNode(args->channel_args)),
+      channel_config_helper_(this),
       work_serializer_(std::make_shared<WorkSerializer>()),
       interested_parties_(grpc_pollset_set_create()),
       subchannel_pool_(GetSubchannelPool(args->channel_args)),
@@ -1451,9 +1716,13 @@ ChannelData::ChannelData(grpc_channel_element_args* args, grpc_error** error)
   channel_args_ = new_args != nullptr
                       ? new_args
                       : grpc_channel_args_copy(args->channel_args);
+  keepalive_time_ = grpc_channel_args_find_integer(
+      channel_args_, GRPC_ARG_KEEPALIVE_TIME_MS,
+      {-1 /* default value, unset */, 1, INT_MAX});
   if (!ResolverRegistry::IsValidTarget(target_uri_.get())) {
-    *error =
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("the target uri is not valid.");
+    std::string error_message =
+        absl::StrCat("the target uri is not valid: ", target_uri_.get());
+    *error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_message.c_str());
     return;
   }
   *error = GRPC_ERROR_NONE;
@@ -1465,6 +1734,7 @@ ChannelData::~ChannelData() {
   }
   DestroyResolvingLoadBalancingPolicyLocked();
   grpc_channel_args_destroy(channel_args_);
+  GRPC_ERROR_UNREF(resolver_transient_failure_error_);
   // Stop backup polling.
   grpc_client_channel_stop_backup_polling(interested_parties_);
   grpc_pollset_set_destroy(interested_parties_);
@@ -1473,16 +1743,18 @@ ChannelData::~ChannelData() {
 }
 
 void ChannelData::UpdateStateAndPickerLocked(
-    grpc_connectivity_state state, const char* reason,
+    grpc_connectivity_state state, const absl::Status& status,
+    const char* reason,
     std::unique_ptr<LoadBalancingPolicy::SubchannelPicker> picker) {
   // Clean the control plane when entering IDLE.
   if (picker_ == nullptr) {
     health_check_service_name_.reset();
     saved_service_config_.reset();
+    saved_config_selector_.reset();
     received_first_resolver_result_ = false;
   }
   // Update connectivity state.
-  state_tracker_.SetState(state, reason);
+  state_tracker_.SetState(state, status, reason);
   if (channelz_node_ != nullptr) {
     channelz_node_->SetConnectivityState(state);
     channelz_node_->AddTraceEvent(
@@ -1501,9 +1773,11 @@ void ChannelData::UpdateStateAndPickerLocked(
   // - refs to subchannel wrappers in the keys of pending_subchannel_updates_
   // - ref stored in retry_throttle_data_
   // - ref stored in service_config_
+  // - ref stored in config_selector_
   // - ownership of the existing picker in picker_
   RefCountedPtr<ServerRetryThrottleData> retry_throttle_data_to_unref;
   RefCountedPtr<ServiceConfig> service_config_to_unref;
+  RefCountedPtr<ConfigSelector> config_selector_to_unref;
   {
     MutexLock lock(&data_plane_mu_);
     // Handle subchannel updates.
@@ -1528,6 +1802,7 @@ void ChannelData::UpdateStateAndPickerLocked(
       // Note: We save the objects to unref until after the lock is released.
       retry_throttle_data_to_unref = std::move(retry_throttle_data_);
       service_config_to_unref = std::move(service_config_);
+      config_selector_to_unref = std::move(config_selector_);
     }
     // Re-process queued picks.
     for (QueuedPick* pick = queued_picks_; pick != nullptr; pick = pick->next) {
@@ -1544,24 +1819,72 @@ void ChannelData::UpdateStateAndPickerLocked(
   pending_subchannel_updates_.clear();
 }
 
-void ChannelData::UpdateServiceConfigLocked(
-    RefCountedPtr<ServerRetryThrottleData> retry_throttle_data,
-    RefCountedPtr<ServiceConfig> service_config) {
+void ChannelData::UpdateServiceConfigInDataPlaneLocked(
+    bool service_config_changed,
+    RefCountedPtr<ConfigSelector> config_selector) {
+  // Check if ConfigSelector has changed.
+  const bool config_selector_changed =
+      saved_config_selector_ != config_selector;
+  saved_config_selector_ = config_selector;
+  // We want to set the service config at least once, even if the
+  // resolver does not return a config, because that ensures that we
+  // disable retries if they are not enabled in the service config.
+  // TODO(roth): Consider removing the received_first_resolver_result_ check
+  // when we implement transparent retries.
+  if (!service_config_changed && !config_selector_changed &&
+      received_first_resolver_result_) {
+    return;
+  }
+  received_first_resolver_result_ = true;
+  // Get retry throttle data from service config.
+  RefCountedPtr<ServerRetryThrottleData> retry_throttle_data;
+  if (saved_service_config_ != nullptr) {
+    const internal::ClientChannelGlobalParsedConfig* parsed_service_config =
+        static_cast<const internal::ClientChannelGlobalParsedConfig*>(
+            saved_service_config_->GetGlobalParsedConfig(
+                internal::ClientChannelServiceConfigParser::ParserIndex()));
+    if (parsed_service_config != nullptr) {
+      absl::optional<internal::ClientChannelGlobalParsedConfig::RetryThrottling>
+          retry_throttle_config = parsed_service_config->retry_throttling();
+      if (retry_throttle_config.has_value()) {
+        retry_throttle_data =
+            internal::ServerRetryThrottleMap::GetDataForServer(
+                server_name_.get(),
+                retry_throttle_config.value().max_milli_tokens,
+                retry_throttle_config.value().milli_token_ratio);
+      }
+    }
+  }
+  // Create default config selector if not provided by resolver.
+  if (config_selector == nullptr) {
+    config_selector =
+        MakeRefCounted<DefaultConfigSelector>(saved_service_config_);
+  }
   // Grab data plane lock to update service config.
   //
   // We defer unreffing the old values (and deallocating memory) until
   // after releasing the lock to keep the critical section small.
+  RefCountedPtr<ServiceConfig> service_config_to_unref = saved_service_config_;
+  RefCountedPtr<ConfigSelector> config_selector_to_unref =
+      std::move(config_selector);
   {
     MutexLock lock(&data_plane_mu_);
+    GRPC_ERROR_UNREF(resolver_transient_failure_error_);
+    resolver_transient_failure_error_ = GRPC_ERROR_NONE;
     // Update service config.
     received_service_config_data_ = true;
     // Old values will be unreffed after lock is released.
     retry_throttle_data_.swap(retry_throttle_data);
-    service_config_.swap(service_config);
-    // Apply service config to queued picks.
+    service_config_.swap(service_config_to_unref);
+    config_selector_.swap(config_selector_to_unref);
+    // Re-process queued picks.
     for (QueuedPick* pick = queued_picks_; pick != nullptr; pick = pick->next) {
-      CallData* calld = static_cast<CallData*>(pick->elem->call_data);
-      calld->MaybeApplyServiceConfigToCallLocked(pick->elem);
+      grpc_call_element* elem = pick->elem;
+      CallData* calld = static_cast<CallData*>(elem->call_data);
+      grpc_error* error = GRPC_ERROR_NONE;
+      if (calld->PickSubchannelLocked(elem, &error)) {
+        calld->AsyncPickDone(elem, error);
+      }
     }
   }
   // Old values will be unreffed after lock is released when they go out
@@ -1578,7 +1901,7 @@ void ChannelData::CreateResolvingLoadBalancingPolicyLocked() {
   grpc_core::UniquePtr<char> target_uri(gpr_strdup(target_uri_.get()));
   resolving_lb_policy_.reset(new ResolvingLoadBalancingPolicy(
       std::move(lb_args), &grpc_client_channel_routing_trace,
-      std::move(target_uri), ProcessResolverResultLocked, this));
+      std::move(target_uri), &channel_config_helper_));
   grpc_pollset_set_add_pollset_set(resolving_lb_policy_->interested_parties(),
                                    interested_parties_);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
@@ -1595,180 +1918,6 @@ void ChannelData::DestroyResolvingLoadBalancingPolicyLocked() {
   }
 }
 
-void ChannelData::ProcessLbPolicy(
-    const Resolver::Result& resolver_result,
-    const internal::ClientChannelGlobalParsedConfig* parsed_service_config,
-    RefCountedPtr<LoadBalancingPolicy::Config>* lb_policy_config) {
-  // Prefer the LB policy config found in the service config.
-  if (parsed_service_config != nullptr &&
-      parsed_service_config->parsed_lb_config() != nullptr) {
-    *lb_policy_config = parsed_service_config->parsed_lb_config();
-    return;
-  }
-  // Try the deprecated LB policy name from the service config.
-  // If not, try the setting from channel args.
-  const char* policy_name = nullptr;
-  if (parsed_service_config != nullptr &&
-      !parsed_service_config->parsed_deprecated_lb_policy().empty()) {
-    policy_name = parsed_service_config->parsed_deprecated_lb_policy().c_str();
-  } else {
-    const grpc_arg* channel_arg =
-        grpc_channel_args_find(resolver_result.args, GRPC_ARG_LB_POLICY_NAME);
-    policy_name = grpc_channel_arg_get_string(channel_arg);
-  }
-  // Use pick_first if nothing was specified and we didn't select grpclb
-  // above.
-  if (policy_name == nullptr) policy_name = "pick_first";
-  // Now that we have the policy name, construct an empty config for it.
-  Json config_json = Json::Array{Json::Object{
-      {policy_name, Json::Object{}},
-  }};
-  grpc_error* parse_error = GRPC_ERROR_NONE;
-  *lb_policy_config = LoadBalancingPolicyRegistry::ParseLoadBalancingConfig(
-      config_json, &parse_error);
-  // The policy name came from one of three places:
-  // - The deprecated loadBalancingPolicy field in the service config,
-  //   in which case the code in ClientChannelServiceConfigParser
-  //   already verified that the policy does not require a config.
-  // - One of the hard-coded values here, all of which are known to not
-  //   require a config.
-  // - A channel arg, in which case the application did something that
-  //   is a misuse of our API.
-  // In the first two cases, these assertions will always be true.  In
-  // the last case, this is probably fine for now.
-  // TODO(roth): If the last case becomes a problem, add better error
-  // handling here.
-  GPR_ASSERT(*lb_policy_config != nullptr);
-  GPR_ASSERT(parse_error == GRPC_ERROR_NONE);
-}
-
-// Synchronous callback from ResolvingLoadBalancingPolicy to process a
-// resolver result update.
-bool ChannelData::ProcessResolverResultLocked(
-    void* arg, const Resolver::Result& result,
-    RefCountedPtr<LoadBalancingPolicy::Config>* lb_policy_config,
-    grpc_error** service_config_error, bool* no_valid_service_config) {
-  ChannelData* chand = static_cast<ChannelData*>(arg);
-  RefCountedPtr<ServiceConfig> service_config;
-  // If resolver did not return a service config or returned an invalid service
-  // config, we need a fallback service config.
-  if (result.service_config_error != GRPC_ERROR_NONE) {
-    // If the service config was invalid, then fallback to the saved service
-    // config. If there is no saved config either, use the default service
-    // config.
-    if (chand->saved_service_config_ != nullptr) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
-        gpr_log(GPR_INFO,
-                "chand=%p: resolver returned invalid service config. "
-                "Continuing to use previous service config.",
-                chand);
-      }
-      service_config = chand->saved_service_config_;
-    } else if (chand->default_service_config_ != nullptr) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
-        gpr_log(GPR_INFO,
-                "chand=%p: resolver returned invalid service config. Using "
-                "default service config provided by client API.",
-                chand);
-      }
-      service_config = chand->default_service_config_;
-    }
-  } else if (result.service_config == nullptr) {
-    if (chand->default_service_config_ != nullptr) {
-      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
-        gpr_log(GPR_INFO,
-                "chand=%p: resolver returned no service config. Using default "
-                "service config provided by client API.",
-                chand);
-      }
-      service_config = chand->default_service_config_;
-    }
-  } else {
-    service_config = result.service_config;
-  }
-  *service_config_error = GRPC_ERROR_REF(result.service_config_error);
-  if (service_config == nullptr &&
-      result.service_config_error != GRPC_ERROR_NONE) {
-    *no_valid_service_config = true;
-    return false;
-  }
-  // Process service config.
-  grpc_core::UniquePtr<char> service_config_json;
-  const internal::ClientChannelGlobalParsedConfig* parsed_service_config =
-      nullptr;
-  if (service_config != nullptr) {
-    parsed_service_config =
-        static_cast<const internal::ClientChannelGlobalParsedConfig*>(
-            service_config->GetGlobalParsedConfig(
-                internal::ClientChannelServiceConfigParser::ParserIndex()));
-  }
-  // Check if the config has changed.
-  const bool service_config_changed =
-      ((service_config == nullptr) !=
-       (chand->saved_service_config_ == nullptr)) ||
-      (service_config != nullptr &&
-       service_config->json_string() !=
-           chand->saved_service_config_->json_string());
-  if (service_config_changed) {
-    service_config_json.reset(gpr_strdup(
-        service_config != nullptr ? service_config->json_string().c_str()
-                                  : ""));
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
-      gpr_log(GPR_INFO,
-              "chand=%p: resolver returned updated service config: \"%s\"",
-              chand, service_config_json.get());
-    }
-    // Save health check service name.
-    if (service_config != nullptr) {
-      chand->health_check_service_name_.reset(
-          gpr_strdup(parsed_service_config->health_check_service_name()));
-    } else {
-      chand->health_check_service_name_.reset();
-    }
-    // Update health check service name used by existing subchannel wrappers.
-    for (auto* subchannel_wrapper : chand->subchannel_wrappers_) {
-      subchannel_wrapper->UpdateHealthCheckServiceName(
-          grpc_core::UniquePtr<char>(
-              gpr_strdup(chand->health_check_service_name_.get())));
-    }
-    // Save service config.
-    chand->saved_service_config_ = std::move(service_config);
-  }
-  // We want to set the service config at least once. This should not really be
-  // needed, but we are doing it as a defensive approach. This can be removed,
-  // if we feel it is unnecessary.
-  if (service_config_changed || !chand->received_first_resolver_result_) {
-    chand->received_first_resolver_result_ = true;
-    RefCountedPtr<ServerRetryThrottleData> retry_throttle_data;
-    if (parsed_service_config != nullptr) {
-      absl::optional<internal::ClientChannelGlobalParsedConfig::RetryThrottling>
-          retry_throttle_config = parsed_service_config->retry_throttling();
-      if (retry_throttle_config.has_value()) {
-        retry_throttle_data =
-            internal::ServerRetryThrottleMap::GetDataForServer(
-                chand->server_name_.get(),
-                retry_throttle_config.value().max_milli_tokens,
-                retry_throttle_config.value().milli_token_ratio);
-      }
-    }
-    chand->UpdateServiceConfigLocked(std::move(retry_throttle_data),
-                                     chand->saved_service_config_);
-  }
-  chand->ProcessLbPolicy(result, parsed_service_config, lb_policy_config);
-  grpc_core::UniquePtr<char> lb_policy_name(
-      gpr_strdup((*lb_policy_config)->name()));
-  // Swap out the data used by GetChannelInfo().
-  {
-    MutexLock lock(&chand->info_mu_);
-    chand->info_lb_policy_name_ = std::move(lb_policy_name);
-    if (service_config_json != nullptr) {
-      chand->info_service_config_json_ = std::move(service_config_json);
-    }
-  }
-  // Return results.
-  return service_config_changed;
-}
-
 grpc_error* ChannelData::DoPingLocked(grpc_transport_op* op) {
   if (state_tracker_.state() != GRPC_CHANNEL_READY) {
     return GRPC_ERROR_CREATE_FROM_STATIC_STRING("channel not connected");
@@ -1832,8 +1981,8 @@ void ChannelData::StartTransportOpLocked(grpc_transport_op* op) {
         static_cast<grpc_connectivity_state>(value) == GRPC_CHANNEL_IDLE) {
       if (disconnect_error() == GRPC_ERROR_NONE) {
         // Enter IDLE state.
-        UpdateStateAndPickerLocked(GRPC_CHANNEL_IDLE, "channel entering IDLE",
-                                   nullptr);
+        UpdateStateAndPickerLocked(GRPC_CHANNEL_IDLE, absl::Status(),
+                                   "channel entering IDLE", nullptr);
       }
       GRPC_ERROR_UNREF(op->disconnect_with_error);
     } else {
@@ -1842,7 +1991,7 @@ void ChannelData::StartTransportOpLocked(grpc_transport_op* op) {
                  GRPC_ERROR_NONE);
       disconnect_error_.Store(op->disconnect_with_error, MemoryOrder::RELEASE);
       UpdateStateAndPickerLocked(
-          GRPC_CHANNEL_SHUTDOWN, "shutdown from API",
+          GRPC_CHANNEL_SHUTDOWN, absl::Status(), "shutdown from API",
           absl::make_unique<LoadBalancingPolicy::TransientFailurePicker>(
               GRPC_ERROR_REF(op->disconnect_with_error)));
     }
@@ -2027,13 +2176,14 @@ void CallData::Destroy(grpc_call_element* elem,
                        const grpc_call_final_info* /*final_info*/,
                        grpc_closure* then_schedule_closure) {
   CallData* calld = static_cast<CallData*>(elem->call_data);
-  if (GPR_LIKELY(calld->subchannel_call_ != nullptr)) {
-    calld->subchannel_call_->SetAfterCallStackDestroy(then_schedule_closure);
-    then_schedule_closure = nullptr;
-  }
+  RefCountedPtr<SubchannelCall> subchannel_call = calld->subchannel_call_;
   calld->~CallData();
-  // TODO(yashkt) : This can potentially be a Closure::Run
-  ExecCtx::Run(DEBUG_LOCATION, then_schedule_closure, GRPC_ERROR_NONE);
+  if (GPR_LIKELY(subchannel_call != nullptr)) {
+    subchannel_call->SetAfterCallStackDestroy(then_schedule_closure);
+  } else {
+    // TODO(yashkt) : This can potentially be a Closure::Run
+    ExecCtx::Run(DEBUG_LOCATION, then_schedule_closure, GRPC_ERROR_NONE);
+  }
 }
 
 void CallData::StartTransportStreamOpBatch(
@@ -2811,6 +2961,7 @@ void CallData::RecvInitialMetadataReady(void* arg, grpc_error* error) {
   }
   // Received valid initial metadata, so commit the call.
   calld->RetryCommit(elem, retry_state);
+  calld->MaybeInvokeConfigSelectorCommitCallback();
   // Invoke the callback to return the result to the surface.
   // Manually invoking a callback function; it does not take ownership of error.
   calld->InvokeRecvInitialMetadataCallback(batch_data, error);
@@ -2897,6 +3048,7 @@ void CallData::RecvMessageReady(void* arg, grpc_error* error) {
   }
   // Received a valid message, so commit the call.
   calld->RetryCommit(elem, retry_state);
+  calld->MaybeInvokeConfigSelectorCommitCallback();
   // Invoke the callback to return the result to the surface.
   // Manually invoking a callback function; it does not take ownership of error.
   calld->InvokeRecvMessageCallback(batch_data, error);
@@ -3098,6 +3250,7 @@ void CallData::RecvTrailingMetadataReady(void* arg, grpc_error* error) {
   }
   // Not retrying, so commit the call.
   calld->RetryCommit(elem, retry_state);
+  calld->MaybeInvokeConfigSelectorCommitCallback();
   // Run any necessary closures.
   calld->RunClosuresForCompletedCall(batch_data, GRPC_ERROR_REF(error));
 }
@@ -3720,7 +3873,7 @@ class CallData::QueuedPickCanceller {
     }
     if (calld->pick_canceller_ == self && error != GRPC_ERROR_NONE) {
       // Remove pick from list of queued picks.
-      calld->RemoveCallFromQueuedPicksLocked(self->elem_);
+      calld->MaybeRemoveCallFromQueuedPicksLocked(self->elem_);
       // Fail pending batches on the call.
       calld->PendingBatchesFail(self->elem_, GRPC_ERROR_REF(error),
                                 YieldCallCombinerIfPendingBatchesFound);
@@ -3733,7 +3886,8 @@ class CallData::QueuedPickCanceller {
   grpc_closure closure_;
 };
 
-void CallData::RemoveCallFromQueuedPicksLocked(grpc_call_element* elem) {
+void CallData::MaybeRemoveCallFromQueuedPicksLocked(grpc_call_element* elem) {
+  if (!pick_queued_) return;
   auto* chand = static_cast<ChannelData*>(elem->channel_data);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: removing from queued picks list",
@@ -3745,7 +3899,8 @@ void CallData::RemoveCallFromQueuedPicksLocked(grpc_call_element* elem) {
   pick_canceller_ = nullptr;
 }
 
-void CallData::AddCallToQueuedPicksLocked(grpc_call_element* elem) {
+void CallData::MaybeAddCallToQueuedPicksLocked(grpc_call_element* elem) {
+  if (pick_queued_) return;
   auto* chand = static_cast<ChannelData*>(elem->channel_data);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: adding to queued picks list", chand,
@@ -3758,23 +3913,29 @@ void CallData::AddCallToQueuedPicksLocked(grpc_call_element* elem) {
   pick_canceller_ = new QueuedPickCanceller(elem);
 }
 
-void CallData::ApplyServiceConfigToCallLocked(grpc_call_element* elem) {
+grpc_error* CallData::ApplyServiceConfigToCallLocked(
+    grpc_call_element* elem, grpc_metadata_batch* initial_metadata) {
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: applying service config to call",
             chand, this);
   }
+  ConfigSelector* config_selector = chand->config_selector();
   auto service_config = chand->service_config();
   if (service_config != nullptr) {
+    // Use the ConfigSelector to determine the config for the call.
+    ConfigSelector::CallConfig call_config =
+        config_selector->GetCallConfig({&path_, initial_metadata, arena_});
+    if (call_config.error != GRPC_ERROR_NONE) return call_config.error;
+    call_attributes_ = std::move(call_config.call_attributes);
+    on_call_committed_ = std::move(call_config.on_call_committed);
     // Create a ServiceConfigCallData for the call.  This stores a ref to the
     // ServiceConfig and caches the right set of parsed configs to use for
     // the call.  The MethodConfig will store itself in the call context,
     // so that it can be accessed by filters in the subchannel, and it
     // will be cleaned up when the call ends.
-    const auto* method_params_vector =
-        service_config->GetMethodParsedConfigVector(path_);
     auto* service_config_call_data = arena_->New<ServiceConfigCallData>(
-        std::move(service_config), method_params_vector, call_context_);
+        std::move(service_config), call_config.method_configs, call_context_);
     // Apply our own method params to the call.
     method_params_ = static_cast<ClientChannelMethodParsedConfig*>(
         service_config_call_data->GetMethodParsedConfig(
@@ -3816,16 +3977,13 @@ void CallData::ApplyServiceConfigToCallLocked(grpc_call_element* elem) {
   if (method_params_ == nullptr || method_params_->retry_policy() == nullptr) {
     enable_retries_ = false;
   }
+  return GRPC_ERROR_NONE;
 }
 
-void CallData::MaybeApplyServiceConfigToCallLocked(grpc_call_element* elem) {
-  ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
-  // Apply service config data to the call only once, and only if the
-  // channel has the data available.
-  if (GPR_LIKELY(chand->received_service_config_data() &&
-                 !service_config_applied_)) {
-    service_config_applied_ = true;
-    ApplyServiceConfigToCallLocked(elem);
+void CallData::MaybeInvokeConfigSelectorCommitCallback() {
+  if (on_call_committed_ != nullptr) {
+    on_call_committed_();
+    on_call_committed_ = nullptr;
   }
 }
 
@@ -3886,11 +4044,45 @@ bool CallData::PickSubchannelLocked(grpc_call_element* elem,
         GRPC_ERROR_NONE);
     // Queue the pick, so that it will be attempted once the channel
     // becomes connected.
-    AddCallToQueuedPicksLocked(elem);
+    MaybeAddCallToQueuedPicksLocked(elem);
     return false;
   }
-  // Apply service config to call if needed.
-  MaybeApplyServiceConfigToCallLocked(elem);
+  grpc_metadata_batch* initial_metadata_batch =
+      seen_send_initial_metadata_
+          ? &send_initial_metadata_
+          : pending_batches_[0]
+                .batch->payload->send_initial_metadata.send_initial_metadata;
+  // Grab initial metadata flags so that we can check later if the call has
+  // wait_for_ready enabled.
+  const uint32_t send_initial_metadata_flags =
+      seen_send_initial_metadata_ ? send_initial_metadata_flags_
+                                  : pending_batches_[0]
+                                        .batch->payload->send_initial_metadata
+                                        .send_initial_metadata_flags;
+  // Avoid picking if we haven't yet received service config data.
+  if (GPR_UNLIKELY(!chand->received_service_config_data())) {
+    // If the resolver returned transient failure before returning the
+    // first service config, fail any non-wait_for_ready calls.
+    grpc_error* resolver_error = chand->resolver_transient_failure_error();
+    if (resolver_error != GRPC_ERROR_NONE &&
+        (send_initial_metadata_flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) ==
+            0) {
+      MaybeRemoveCallFromQueuedPicksLocked(elem);
+      *error = GRPC_ERROR_REF(resolver_error);
+      return true;
+    }
+    // Either the resolver has not yet returned a result, or it has
+    // returned transient failure but the call is wait_for_ready.  In
+    // either case, queue the call.
+    MaybeAddCallToQueuedPicksLocked(elem);
+    return false;
+  }
+  // Apply service config to call if not yet applied.
+  if (GPR_LIKELY(!service_config_applied_)) {
+    service_config_applied_ = true;
+    *error = ApplyServiceConfigToCallLocked(elem, initial_metadata_batch);
+    if (*error != GRPC_ERROR_NONE) return true;
+  }
   // If this is a retry, use the send_initial_metadata payload that
   // we've cached; otherwise, use the pending batch.  The
   // send_initial_metadata batch will be the first pending batch in the
@@ -3902,21 +4094,10 @@ bool CallData::PickSubchannelLocked(grpc_call_element* elem,
   // subchannel's copy of the metadata batch (which is copied for each
   // attempt) to the LB policy instead the one from the parent channel.
   LoadBalancingPolicy::PickArgs pick_args;
+  pick_args.path = StringViewFromSlice(path_);
   pick_args.call_state = &lb_call_state_;
-  Metadata initial_metadata(
-      this,
-      seen_send_initial_metadata_
-          ? &send_initial_metadata_
-          : pending_batches_[0]
-                .batch->payload->send_initial_metadata.send_initial_metadata);
+  Metadata initial_metadata(this, initial_metadata_batch);
   pick_args.initial_metadata = &initial_metadata;
-  // Grab initial metadata flags so that we can check later if the call has
-  // wait_for_ready enabled.
-  const uint32_t send_initial_metadata_flags =
-      seen_send_initial_metadata_ ? send_initial_metadata_flags_
-                                  : pending_batches_[0]
-                                        .batch->payload->send_initial_metadata
-                                        .send_initial_metadata_flags;
   // Attempt pick.
   auto result = chand->picker()->Pick(pick_args);
   if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
@@ -3931,7 +4112,8 @@ bool CallData::PickSubchannelLocked(grpc_call_element* elem,
       grpc_error* disconnect_error = chand->disconnect_error();
       if (disconnect_error != GRPC_ERROR_NONE) {
         GRPC_ERROR_UNREF(result.error);
-        if (pick_queued_) RemoveCallFromQueuedPicksLocked(elem);
+        MaybeRemoveCallFromQueuedPicksLocked(elem);
+        MaybeInvokeConfigSelectorCommitCallback();
         *error = GRPC_ERROR_REF(disconnect_error);
         return true;
       }
@@ -3952,8 +4134,9 @@ bool CallData::PickSubchannelLocked(grpc_call_element* elem,
                   "Failed to pick subchannel", &result.error, 1);
           GRPC_ERROR_UNREF(result.error);
           *error = new_error;
+          MaybeInvokeConfigSelectorCommitCallback();
         }
-        if (pick_queued_) RemoveCallFromQueuedPicksLocked(elem);
+        MaybeRemoveCallFromQueuedPicksLocked(elem);
         return !retried;
       }
       // If wait_for_ready is true, then queue to retry when we get a new
@@ -3962,22 +4145,24 @@ bool CallData::PickSubchannelLocked(grpc_call_element* elem,
     }
     // Fallthrough
     case LoadBalancingPolicy::PickResult::PICK_QUEUE:
-      if (!pick_queued_) AddCallToQueuedPicksLocked(elem);
+      MaybeAddCallToQueuedPicksLocked(elem);
       return false;
     default:  // PICK_COMPLETE
-      if (pick_queued_) RemoveCallFromQueuedPicksLocked(elem);
+      MaybeRemoveCallFromQueuedPicksLocked(elem);
       // Handle drops.
       if (GPR_UNLIKELY(result.subchannel == nullptr)) {
         result.error = grpc_error_set_int(
             GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                 "Call dropped by load balancing policy"),
             GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+        MaybeInvokeConfigSelectorCommitCallback();
       } else {
         // Grab a ref to the connected subchannel while we're still
         // holding the data plane mutex.
         connected_subchannel_ =
             chand->GetConnectedSubchannelInDataPlane(result.subchannel.get());
         GPR_ASSERT(connected_subchannel_ != nullptr);
+        if (retry_committed_) MaybeInvokeConfigSelectorCommitCallback();
       }
       lb_recv_trailing_metadata_ready_ = result.recv_trailing_metadata_ready;
       *error = result.error;