grpc 1.32.0 → 1.33.0.pre1

data/src/core/ext/filters/client_channel/lb_policy_registry.cc

@@ -23,7 +23,9 @@
 #include <string.h>
 
 #include "absl/container/inlined_vector.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
 
 #include "src/core/lib/gpr/string.h"
 
@@ -126,6 +128,7 @@ grpc_error* ParseLoadBalancingConfigHelper(
     return GRPC_ERROR_CREATE_FROM_STATIC_STRING("type should be array");
   }
   // Find the first LB policy that this client supports.
+  std::vector<absl::string_view> policies_tried;
   for (const Json& lb_config : lb_config_array.array_value()) {
     if (lb_config.type() != Json::Type::OBJECT) {
       return GRPC_ERROR_CREATE_FROM_STATIC_STRING(
@@ -149,8 +152,12 @@ grpc_error* ParseLoadBalancingConfigHelper(
       *result = it;
       return GRPC_ERROR_NONE;
     }
+    policies_tried.push_back(it->first);
   }
-  return GRPC_ERROR_CREATE_FROM_STATIC_STRING("No known policy");
+  return GRPC_ERROR_CREATE_FROM_COPIED_STRING(
+      absl::StrCat("No known policies in list: ",
+                   absl::StrJoin(policies_tried, " "))
+          .c_str());
 }
 
 }  // namespace

data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc

@@ -352,7 +352,7 @@ void AresDnsResolver::OnResolvedLocked(grpc_error* error) {
         GRPC_CARES_TRACE_LOG("resolver:%p selected service config choice: %s",
                              this, service_config_string.c_str());
         result.service_config = ServiceConfig::Create(
-            service_config_string, &result.service_config_error);
+            channel_args_, service_config_string, &result.service_config_error);
       }
     }
     absl::InlinedVector<grpc_arg, 1> new_args;

data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc

@@ -18,18 +18,25 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "absl/strings/match.h"
 #include "absl/strings/str_join.h"
+#include "absl/strings/str_split.h"
+#include "re2/re2.h"
 
 #include "src/core/ext/filters/client_channel/config_selector.h"
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/ext/xds/xds_client.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/transport/timeout_encoding.h"
 
 namespace grpc_core {
 
 TraceFlag grpc_xds_resolver_trace(false, "xds_resolver");
 
+const char* kXdsClusterAttribute = "xds_cluster_name";
+
 namespace {
 
 //
@@ -42,8 +49,7 @@ class XdsResolver : public Resolver {
       : Resolver(std::move(args.work_serializer),
                  std::move(args.result_handler)),
         args_(grpc_channel_args_copy(args.args)),
-        interested_parties_(args.pollset_set),
-        config_selector_(MakeRefCounted<XdsConfigSelector>()) {
+        interested_parties_(args.pollset_set) {
     char* path = args.uri->path;
     if (path[0] == '/') ++path;
     server_name_ = path;
@@ -62,124 +68,443 @@ class XdsResolver : public Resolver {
 
   void StartLocked() override;
 
-  void ShutdownLocked() override {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) {
-      gpr_log(GPR_INFO, "[xds_resolver %p] shutting down", this);
-    }
-    xds_client_.reset();
-  }
+  void ShutdownLocked() override;
 
  private:
+  class Notifier {
+   public:
+    Notifier(RefCountedPtr<XdsResolver> resolver, XdsApi::LdsUpdate update);
+    Notifier(RefCountedPtr<XdsResolver> resolver, XdsApi::RdsUpdate update);
+    Notifier(RefCountedPtr<XdsResolver> resolver, grpc_error* error);
+    explicit Notifier(RefCountedPtr<XdsResolver> resolver);
+
+   private:
+    enum Type { kLdsUpdate, kRdsUpdate, kError, kDoesNotExist };
+
+    static void RunInExecCtx(void* arg, grpc_error* error);
+    void RunInWorkSerializer(grpc_error* error);
+
+    RefCountedPtr<XdsResolver> resolver_;
+    grpc_closure closure_;
+    XdsApi::LdsUpdate update_;
+    Type type_;
+  };
+
   class ListenerWatcher : public XdsClient::ListenerWatcherInterface {
    public:
     explicit ListenerWatcher(RefCountedPtr<XdsResolver> resolver)
         : resolver_(std::move(resolver)) {}
-    void OnListenerChanged(std::vector<XdsApi::Route> routes) override;
-    void OnError(grpc_error* error) override;
-    void OnResourceDoesNotExist() override;
+    void OnListenerChanged(XdsApi::LdsUpdate listener) override {
+      new Notifier(resolver_, std::move(listener));
+    }
+    void OnError(grpc_error* error) override { new Notifier(resolver_, error); }
+    void OnResourceDoesNotExist() override { new Notifier(resolver_); }
 
    private:
     RefCountedPtr<XdsResolver> resolver_;
   };
 
-  class XdsConfigSelector : public ConfigSelector {
+  class RouteConfigWatcher : public XdsClient::RouteConfigWatcherInterface {
    public:
-    CallConfig GetCallConfig(GetCallConfigArgs args) override {
-      return CallConfig();
+    explicit RouteConfigWatcher(RefCountedPtr<XdsResolver> resolver)
+        : resolver_(std::move(resolver)) {}
+    void OnRouteConfigChanged(XdsApi::RdsUpdate route_config) override {
+      new Notifier(resolver_, std::move(route_config));
     }
+    void OnError(grpc_error* error) override { new Notifier(resolver_, error); }
+    void OnResourceDoesNotExist() override { new Notifier(resolver_); }
+
+   private:
+    RefCountedPtr<XdsResolver> resolver_;
+  };
+
+  class ClusterState
+      : public RefCounted<ClusterState, PolymorphicRefCount, false> {
+   public:
+    using ClusterStateMap =
+        std::map<std::string, std::unique_ptr<ClusterState>>;
+
+    ClusterState(const std::string& cluster_name,
+                 ClusterStateMap* cluster_state_map)
+        : it_(cluster_state_map
+                  ->emplace(cluster_name, std::unique_ptr<ClusterState>(this))
+                  .first) {}
+    const std::string& cluster() const { return it_->first; }
+
+   private:
+    ClusterStateMap::iterator it_;
   };
 
-  // Returns the weighted_clusters action name to use from
-  // weighted_cluster_index_map_ for a WeightedClusters route action.
-  std::string WeightedClustersActionName(
-      const std::vector<XdsApi::Route::ClusterWeight>& weighted_clusters);
+  class XdsConfigSelector : public ConfigSelector {
+   public:
+    XdsConfigSelector(RefCountedPtr<XdsResolver> resolver,
+                      const std::vector<XdsApi::Route>& routes);
+    ~XdsConfigSelector();
+
+    const char* name() const override { return "XdsConfigSelector"; }
+
+    bool Equals(const ConfigSelector* other) const override {
+      const auto* other_xds = static_cast<const XdsConfigSelector*>(other);
+      // Don't need to compare resolver_, since that will always be the same.
+      return route_table_ == other_xds->route_table_ &&
+             clusters_ == other_xds->clusters_;
+    }
+
+    CallConfig GetCallConfig(GetCallConfigArgs args) override;
+
+   private:
+    struct Route {
+      XdsApi::Route route;
+      absl::InlinedVector<std::pair<uint32_t, absl::string_view>, 2>
+          weighted_cluster_state;
+      bool operator==(const Route& other) const {
+        return route == other.route &&
+               weighted_cluster_state == other.weighted_cluster_state;
+      }
+    };
+    using RouteTable = std::vector<Route>;
+
+    void MaybeAddCluster(const std::string& name);
+
+    RefCountedPtr<XdsResolver> resolver_;
+    RouteTable route_table_;
+    std::map<absl::string_view, RefCountedPtr<ClusterState>> clusters_;
+  };
 
-  // Updates weighted_cluster_index_map_ that will
-  // determine the names of the WeightedCluster actions for the current update.
-  void UpdateWeightedClusterIndexMap(const std::vector<XdsApi::Route>& routes);
+  void OnListenerUpdate(XdsApi::LdsUpdate listener);
+  void OnRouteConfigUpdate(XdsApi::RdsUpdate rds_update);
+  void OnError(grpc_error* error);
+  void OnResourceDoesNotExist();
 
-  // Create the service config generated by the list of routes.
-  grpc_error* CreateServiceConfig(const std::vector<XdsApi::Route>& routes,
-                                  RefCountedPtr<ServiceConfig>* service_config);
+  grpc_error* CreateServiceConfig(RefCountedPtr<ServiceConfig>* service_config);
+  void GenerateResult();
+  void MaybeRemoveUnusedClusters();
 
   std::string server_name_;
   const grpc_channel_args* args_;
   grpc_pollset_set* interested_parties_;
-  OrphanablePtr<XdsClient> xds_client_;
-  RefCountedPtr<XdsConfigSelector> config_selector_;
-
-  // 2-level map to store WeightedCluster action names.
-  // Top level map is keyed by cluster names without weight like a_b_c; bottom
-  // level map is keyed by cluster names + weights like a10_b50_c40.
-  struct ClusterNamesInfo {
-    uint64_t next_index = 0;
-    std::map<std::string /*cluster names + weights*/,
-             uint64_t /*policy index number*/>
-        cluster_weights_map;
-  };
-  using WeightedClusterIndexMap =
-      std::map<std::string /*cluster names*/, ClusterNamesInfo>;
-
-  // Cache of action names for WeightedCluster targets in the current
-  // service config.
-  WeightedClusterIndexMap weighted_cluster_index_map_;
+  RefCountedPtr<XdsClient> xds_client_;
+  XdsClient::ListenerWatcherInterface* listener_watcher_ = nullptr;
+  std::string route_config_name_;
+  XdsClient::RouteConfigWatcherInterface* route_config_watcher_ = nullptr;
+  ClusterState::ClusterStateMap cluster_state_map_;
+  std::vector<XdsApi::Route> current_update_;
 };
 
 //
-// XdsResolver::ListenerWatcher
+// XdsResolver::Notifier
 //
 
-void XdsResolver::ListenerWatcher::OnListenerChanged(
-    std::vector<XdsApi::Route> routes) {
-  if (resolver_->xds_client_ == nullptr) return;
+XdsResolver::Notifier::Notifier(RefCountedPtr<XdsResolver> resolver,
+                                XdsApi::LdsUpdate update)
+    : resolver_(std::move(resolver)),
+      update_(std::move(update)),
+      type_(kLdsUpdate) {
+  GRPC_CLOSURE_INIT(&closure_, &RunInExecCtx, this, nullptr);
+  ExecCtx::Run(DEBUG_LOCATION, &closure_, GRPC_ERROR_NONE);
+}
+
+XdsResolver::Notifier::Notifier(RefCountedPtr<XdsResolver> resolver,
+                                XdsApi::RdsUpdate update)
+    : resolver_(std::move(resolver)), type_(kRdsUpdate) {
+  update_.rds_update = std::move(update);
+  GRPC_CLOSURE_INIT(&closure_, &RunInExecCtx, this, nullptr);
+  ExecCtx::Run(DEBUG_LOCATION, &closure_, GRPC_ERROR_NONE);
+}
+
+XdsResolver::Notifier::Notifier(RefCountedPtr<XdsResolver> resolver,
+                                grpc_error* error)
+    : resolver_(std::move(resolver)), type_(kError) {
+  GRPC_CLOSURE_INIT(&closure_, &RunInExecCtx, this, nullptr);
+  ExecCtx::Run(DEBUG_LOCATION, &closure_, error);
+}
+
+XdsResolver::Notifier::Notifier(RefCountedPtr<XdsResolver> resolver)
+    : resolver_(std::move(resolver)), type_(kDoesNotExist) {
+  GRPC_CLOSURE_INIT(&closure_, &RunInExecCtx, this, nullptr);
+  ExecCtx::Run(DEBUG_LOCATION, &closure_, GRPC_ERROR_NONE);
+}
+
+void XdsResolver::Notifier::RunInExecCtx(void* arg, grpc_error* error) {
+  Notifier* self = static_cast<Notifier*>(arg);
+  GRPC_ERROR_REF(error);
+  self->resolver_->work_serializer()->Run(
+      [self, error]() { self->RunInWorkSerializer(error); }, DEBUG_LOCATION);
+}
+
+void XdsResolver::Notifier::RunInWorkSerializer(grpc_error* error) {
+  if (resolver_->xds_client_ == nullptr) {
+    GRPC_ERROR_UNREF(error);
+    delete this;
+    return;
+  }
+  switch (type_) {
+    case kLdsUpdate:
+      resolver_->OnListenerUpdate(std::move(update_));
+      break;
+    case kRdsUpdate:
+      resolver_->OnRouteConfigUpdate(std::move(*update_.rds_update));
+      break;
+    case kError:
+      resolver_->OnError(error);
+      break;
+    case kDoesNotExist:
+      resolver_->OnResourceDoesNotExist();
+      break;
+  };
+  delete this;
+}
+
+//
+// XdsResolver::XdsConfigSelector
+//
+
+XdsResolver::XdsConfigSelector::XdsConfigSelector(
+    RefCountedPtr<XdsResolver> resolver,
+    const std::vector<XdsApi::Route>& routes)
+    : resolver_(std::move(resolver)) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) {
-    gpr_log(GPR_INFO, "[xds_resolver %p] received updated listener data",
-            resolver_.get());
+    gpr_log(GPR_INFO, "[xds_resolver %p] creating XdsConfigSelector %p",
+            resolver_.get(), this);
   }
-  Result result;
-  grpc_error* error =
-      resolver_->CreateServiceConfig(routes, &result.service_config);
-  if (error != GRPC_ERROR_NONE) {
-    OnError(error);
-    return;
+  // 1. Construct the route table
+  // 2  Update resolver's cluster state map
+  // 3. Construct cluster list to hold on to entries in the cluster state
+  // map.
+  // Reserve the necessary entries up-front to avoid reallocation as we add
+  // elements. This is necessary because the string_view in the entry's
+  // weighted_cluster_state field points to the memory in the route field, so
+  // moving the entry in a reallocation will cause the string_view to point to
+  // invalid data.
+  route_table_.reserve(routes.size());
+  for (auto& route : routes) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) {
+      gpr_log(GPR_INFO, "[xds_resolver %p] XdsConfigSelector %p: route: %s",
+              resolver_.get(), this, route.ToString().c_str());
+    }
+    route_table_.emplace_back();
+    auto& route_entry = route_table_.back();
+    route_entry.route = route;
+    if (route.weighted_clusters.empty()) {
+      MaybeAddCluster(route.cluster_name);
+    } else {
+      uint32_t end = 0;
+      for (const auto& weighted_cluster : route_entry.route.weighted_clusters) {
+        MaybeAddCluster(weighted_cluster.name);
+        end += weighted_cluster.weight;
+        route_entry.weighted_cluster_state.emplace_back(end,
+                                                        weighted_cluster.name);
+      }
+    }
   }
+}
+
+XdsResolver::XdsConfigSelector::~XdsConfigSelector() {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) {
-    gpr_log(GPR_INFO, "[xds_resolver %p] generated service config: %s",
-            resolver_.get(), result.service_config->json_string().c_str());
+    gpr_log(GPR_INFO, "[xds_resolver %p] destroying XdsConfigSelector %p",
+            resolver_.get(), this);
   }
-  grpc_arg new_args[] = {
-      resolver_->xds_client_->MakeChannelArg(),
-      resolver_->config_selector_->MakeChannelArg(),
-  };
-  result.args = grpc_channel_args_copy_and_add(resolver_->args_, new_args,
-                                               GPR_ARRAY_SIZE(new_args));
-  resolver_->result_handler()->ReturnResult(std::move(result));
+  clusters_.clear();
+  resolver_->MaybeRemoveUnusedClusters();
 }
 
-void XdsResolver::ListenerWatcher::OnError(grpc_error* error) {
-  if (resolver_->xds_client_ == nullptr) return;
-  gpr_log(GPR_ERROR, "[xds_resolver %p] received error: %s", resolver_.get(),
-          grpc_error_string(error));
-  grpc_arg xds_client_arg = resolver_->xds_client_->MakeChannelArg();
-  Result result;
-  result.args =
-      grpc_channel_args_copy_and_add(resolver_->args_, &xds_client_arg, 1);
-  result.service_config_error = error;
-  resolver_->result_handler()->ReturnResult(std::move(result));
+void XdsResolver::XdsConfigSelector::MaybeAddCluster(const std::string& name) {
+  if (clusters_.find(name) == clusters_.end()) {
+    auto it = resolver_->cluster_state_map_.find(name);
+    if (it == resolver_->cluster_state_map_.end()) {
+      auto new_cluster_state =
+          MakeRefCounted<ClusterState>(name, &resolver_->cluster_state_map_);
+      clusters_[new_cluster_state->cluster()] = std::move(new_cluster_state);
+    } else {
+      clusters_[it->second->cluster()] = it->second->Ref();
+    }
+  }
 }
 
-void XdsResolver::ListenerWatcher::OnResourceDoesNotExist() {
-  if (resolver_->xds_client_ == nullptr) return;
-  gpr_log(GPR_ERROR,
-          "[xds_resolver %p] LDS/RDS resource does not exist -- returning "
-          "empty service config",
-          resolver_.get());
-  Result result;
-  result.service_config =
-      ServiceConfig::Create("{}", &result.service_config_error);
-  GPR_ASSERT(result.service_config != nullptr);
-  result.args = grpc_channel_args_copy(resolver_->args_);
-  resolver_->result_handler()->ReturnResult(std::move(result));
+bool PathMatch(const absl::string_view& path,
+               const XdsApi::Route::Matchers::PathMatcher& path_matcher) {
+  switch (path_matcher.type) {
+    case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PREFIX:
+      return absl::StartsWith(path, path_matcher.string_matcher);
+    case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PATH:
+      return path == path_matcher.string_matcher;
+    case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::REGEX:
+      return RE2::FullMatch(path.data(), *path_matcher.regex_matcher);
+    default:
+      return false;
+  }
+}
+
+absl::optional<absl::string_view> GetMetadataValue(
+    const std::string& target_key, grpc_metadata_batch* initial_metadata,
+    std::string* concatenated_value) {
+  // Find all values for the specified key.
+  GPR_DEBUG_ASSERT(initial_metadata != nullptr);
+  absl::InlinedVector<absl::string_view, 1> values;
+  for (grpc_linked_mdelem* md = initial_metadata->list.head; md != nullptr;
+       md = md->next) {
+    absl::string_view key = StringViewFromSlice(GRPC_MDKEY(md->md));
+    absl::string_view value = StringViewFromSlice(GRPC_MDVALUE(md->md));
+    if (target_key == key) values.push_back(value);
+  }
+  // If none found, no match.
+  if (values.empty()) return absl::nullopt;
+  // If exactly one found, return it as-is.
+  if (values.size() == 1) return values.front();
+  // If more than one found, concatenate the values, using
+  // *concatenated_values as a temporary holding place for the
+  // concatenated string.
+  *concatenated_value = absl::StrJoin(values, ",");
+  return *concatenated_value;
+}
+
+bool HeaderMatchHelper(
+    const XdsApi::Route::Matchers::HeaderMatcher& header_matcher,
+    grpc_metadata_batch* initial_metadata) {
+  std::string concatenated_value;
+  absl::optional<absl::string_view> value;
+  // Note: If we ever allow binary headers here, we still need to
+  // special-case ignore "grpc-tags-bin" and "grpc-trace-bin", since
+  // they are not visible to the LB policy in grpc-go.
+  if (absl::EndsWith(header_matcher.name, "-bin") ||
+      header_matcher.name == "grpc-previous-rpc-attempts") {
+    value = absl::nullopt;
+  } else if (header_matcher.name == "content-type") {
+    value = "application/grpc";
+  } else {
+    value = GetMetadataValue(header_matcher.name, initial_metadata,
+                             &concatenated_value);
+  }
+  if (!value.has_value()) {
+    if (header_matcher.type ==
+        XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PRESENT) {
+      return !header_matcher.present_match;
+    } else {
+      // For all other header matcher types, we need the header value to
+      // exist to consider matches.
+      return false;
+    }
+  }
+  switch (header_matcher.type) {
+    case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::EXACT:
+      return value.value() == header_matcher.string_matcher;
+    case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::REGEX:
+      return RE2::FullMatch(value.value().data(), *header_matcher.regex_match);
+    case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::RANGE:
+      int64_t int_value;
+      if (!absl::SimpleAtoi(value.value(), &int_value)) {
+        return false;
+      }
+      return int_value >= header_matcher.range_start &&
+             int_value < header_matcher.range_end;
+    case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PREFIX:
+      return absl::StartsWith(value.value(), header_matcher.string_matcher);
+    case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::SUFFIX:
+      return absl::EndsWith(value.value(), header_matcher.string_matcher);
+    default:
+      return false;
+  }
+}
+
+bool HeadersMatch(
+    const std::vector<XdsApi::Route::Matchers::HeaderMatcher>& header_matchers,
+    grpc_metadata_batch* initial_metadata) {
+  for (const auto& header_matcher : header_matchers) {
+    bool match = HeaderMatchHelper(header_matcher, initial_metadata);
+    if (header_matcher.invert_match) match = !match;
+    if (!match) return false;
+  }
+  return true;
+}
+
+bool UnderFraction(const uint32_t fraction_per_million) {
+  // Generate a random number in [0, 1000000).
+  const uint32_t random_number = rand() % 1000000;
+  return random_number < fraction_per_million;
+}
+
+ConfigSelector::CallConfig XdsResolver::XdsConfigSelector::GetCallConfig(
+    GetCallConfigArgs args) {
+  for (const auto& entry : route_table_) {
+    // Path matching.
+    if (!PathMatch(StringViewFromSlice(*args.path),
+                   entry.route.matchers.path_matcher)) {
+      continue;
+    }
+    // Header Matching.
+    if (!HeadersMatch(entry.route.matchers.header_matchers,
+                      args.initial_metadata)) {
+      continue;
+    }
+    // Match fraction check
+    if (entry.route.matchers.fraction_per_million.has_value() &&
+        !UnderFraction(entry.route.matchers.fraction_per_million.value())) {
+      continue;
+    }
+    // Found a route match
+    absl::string_view cluster_name;
+    if (entry.route.weighted_clusters.empty()) {
+      cluster_name = entry.route.cluster_name;
+    } else {
+      const uint32_t key =
+          rand() %
+          entry.weighted_cluster_state[entry.weighted_cluster_state.size() - 1]
+              .first;
+      // Find the index in weighted clusters corresponding to key.
+      size_t mid = 0;
+      size_t start_index = 0;
+      size_t end_index = entry.weighted_cluster_state.size() - 1;
+      size_t index = 0;
+      while (end_index > start_index) {
+        mid = (start_index + end_index) / 2;
+        if (entry.weighted_cluster_state[mid].first > key) {
+          end_index = mid;
+        } else if (entry.weighted_cluster_state[mid].first < key) {
+          start_index = mid + 1;
+        } else {
+          index = mid + 1;
+          break;
+        }
+      }
+      if (index == 0) index = start_index;
+      GPR_ASSERT(entry.weighted_cluster_state[index].first > key);
+      cluster_name = entry.weighted_cluster_state[index].second;
+    }
+    auto it = clusters_.find(cluster_name);
+    GPR_ASSERT(it != clusters_.end());
+    XdsResolver* resolver =
+        static_cast<XdsResolver*>(resolver_->Ref().release());
+    ClusterState* cluster_state = it->second->Ref().release();
+    CallConfig call_config;
+    call_config.call_attributes[kXdsClusterAttribute] = it->first;
+    call_config.on_call_committed = [resolver, cluster_state]() {
+      cluster_state->Unref();
+      ExecCtx::Run(
+          // TODO(roth): This hop into the ExecCtx is being done to avoid
+          // entering the WorkSerializer while holding the client channel data
+          // plane mutex, since that can lead to deadlocks. However, we should
+          // not have to solve this problem in each individual ConfigSelector
+          // implementation. When we have time, we should fix the client channel
+          // code to avoid this by not invoking the
+          // CallConfig::on_call_committed callback until after it has released
+          // the data plane mutex.
+          DEBUG_LOCATION,
+          GRPC_CLOSURE_CREATE(
+              [](void* arg, grpc_error* /*error*/) {
+                auto* resolver = static_cast<XdsResolver*>(arg);
+                resolver->work_serializer()->Run(
+                    [resolver]() {
+                      resolver->MaybeRemoveUnusedClusters();
+                      resolver->Unref();
+                    },
+                    DEBUG_LOCATION);
+              },
+              resolver, nullptr),
+          GRPC_ERROR_NONE);
+    };
+    return call_config;
+  }
+  return CallConfig();
 }
 
 //
@@ -188,320 +513,190 @@ void XdsResolver::ListenerWatcher::OnResourceDoesNotExist() {
 
 void XdsResolver::StartLocked() {
   grpc_error* error = GRPC_ERROR_NONE;
-  xds_client_ = MakeOrphanable<XdsClient>(
-      work_serializer(), interested_parties_, server_name_,
-      absl::make_unique<ListenerWatcher>(Ref()), *args_, &error);
+  xds_client_ = XdsClient::GetOrCreate(&error);
   if (error != GRPC_ERROR_NONE) {
     gpr_log(GPR_ERROR,
             "Failed to create xds client -- channel will remain in "
             "TRANSIENT_FAILURE: %s",
             grpc_error_string(error));
     result_handler()->ReturnError(error);
+    return;
+  }
+  grpc_pollset_set_add_pollset_set(xds_client_->interested_parties(),
+                                   interested_parties_);
+  channelz::ChannelNode* parent_channelz_node =
+      grpc_channel_args_find_pointer<channelz::ChannelNode>(
+          args_, GRPC_ARG_CHANNELZ_CHANNEL_NODE);
+  if (parent_channelz_node != nullptr) {
+    xds_client_->AddChannelzLinkage(parent_channelz_node);
   }
+  auto watcher = absl::make_unique<ListenerWatcher>(Ref());
+  listener_watcher_ = watcher.get();
+  xds_client_->WatchListenerData(server_name_, std::move(watcher));
 }
 
-std::string CreateServiceConfigActionCluster(const std::string& cluster_name) {
-  return absl::StrFormat(
-      "      \"cds:%s\":{\n"
-      "        \"childPolicy\":[ {\n"
-      "          \"cds_experimental\":{\n"
-      "            \"cluster\": \"%s\"\n"
-      "          }\n"
-      "        } ]\n"
-      "       }",
-      cluster_name, cluster_name);
+void XdsResolver::ShutdownLocked() {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) {
+    gpr_log(GPR_INFO, "[xds_resolver %p] shutting down", this);
+  }
+  if (xds_client_ != nullptr) {
+    if (listener_watcher_ != nullptr) {
+      xds_client_->CancelListenerDataWatch(server_name_, listener_watcher_,
+                                           /*delay_unsubscription=*/false);
+    }
+    if (route_config_watcher_ != nullptr) {
+      xds_client_->CancelRouteConfigDataWatch(
+          server_name_, route_config_watcher_, /*delay_unsubscription=*/false);
+    }
+    channelz::ChannelNode* parent_channelz_node =
+        grpc_channel_args_find_pointer<channelz::ChannelNode>(
+            args_, GRPC_ARG_CHANNELZ_CHANNEL_NODE);
+    if (parent_channelz_node != nullptr) {
+      xds_client_->RemoveChannelzLinkage(parent_channelz_node);
+    }
+    grpc_pollset_set_del_pollset_set(xds_client_->interested_parties(),
+                                     interested_parties_);
+    xds_client_.reset();
+  }
 }
 
-std::string CreateServiceConfigRoute(const std::string& action_name,
-                                     const XdsApi::Route& route) {
-  std::vector<std::string> headers;
-  for (const auto& header : route.matchers.header_matchers) {
-    std::string header_matcher;
-    switch (header.type) {
-      case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::EXACT:
-        header_matcher = absl::StrFormat("             \"exact_match\": \"%s\"",
-                                         header.string_matcher);
-        break;
-      case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::REGEX:
-        header_matcher = absl::StrFormat("             \"regex_match\": \"%s\"",
-                                         header.regex_match->pattern());
-        break;
-      case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::RANGE:
-        header_matcher = absl::StrFormat(
-            "             \"range_match\":{\n"
-            "              \"start\":%d,\n"
-            "              \"end\":%d\n"
-            "             }",
-            header.range_start, header.range_end);
-        break;
-      case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PRESENT:
-        header_matcher =
-            absl::StrFormat("             \"present_match\": %s",
-                            header.present_match ? "true" : "false");
-        break;
-      case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PREFIX:
-        header_matcher = absl::StrFormat(
-            "             \"prefix_match\": \"%s\"", header.string_matcher);
-        break;
-      case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::SUFFIX:
-        header_matcher = absl::StrFormat(
-            "             \"suffix_match\": \"%s\"", header.string_matcher);
-        break;
-      default:
-        break;
+void XdsResolver::OnListenerUpdate(XdsApi::LdsUpdate listener) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) {
+    gpr_log(GPR_INFO, "[xds_resolver %p] received updated listener data", this);
+  }
+  if (listener.route_config_name != route_config_name_) {
+    if (route_config_watcher_ != nullptr) {
+      xds_client_->CancelRouteConfigDataWatch(
+          route_config_name_, route_config_watcher_,
+          /*delay_unsubscription=*/!listener.route_config_name.empty());
+      route_config_watcher_ = nullptr;
     }
-    std::vector<std::string> header_parts;
-    header_parts.push_back(
-        absl::StrFormat("           { \n"
-                        "             \"name\": \"%s\",\n",
-                        header.name));
-    header_parts.push_back(header_matcher);
-    if (header.invert_match) {
-      header_parts.push_back(
-          absl::StrFormat(",\n"
-                          "             \"invert_match\": true"));
+    route_config_name_ = std::move(listener.route_config_name);
+    if (!route_config_name_.empty()) {
+      auto watcher = absl::make_unique<RouteConfigWatcher>(Ref());
+      route_config_watcher_ = watcher.get();
+      xds_client_->WatchRouteConfigData(route_config_name_, std::move(watcher));
     }
-    header_parts.push_back(
-        absl::StrFormat("\n"
-                        "           }"));
-    headers.push_back(absl::StrJoin(header_parts, ""));
   }
-  std::vector<std::string> headers_service_config;
-  if (!headers.empty()) {
-    headers_service_config.push_back("\"headers\":[\n");
-    headers_service_config.push_back(absl::StrJoin(headers, ","));
-    headers_service_config.push_back("           ],\n");
+  if (route_config_name_.empty()) {
+    GPR_ASSERT(listener.rds_update.has_value());
+    OnRouteConfigUpdate(std::move(*listener.rds_update));
   }
-  std::string path_match_str;
-  switch (route.matchers.path_matcher.type) {
-    case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PREFIX:
-      path_match_str = absl::StrFormat(
-          "\"prefix\": \"%s\",\n", route.matchers.path_matcher.string_matcher);
-      break;
-    case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PATH:
-      path_match_str = absl::StrFormat(
-          "\"path\": \"%s\",\n", route.matchers.path_matcher.string_matcher);
-      break;
-    case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::REGEX:
-      path_match_str =
-          absl::StrFormat("\"regex\": \"%s\",\n",
-                          route.matchers.path_matcher.regex_matcher->pattern());
-      break;
-  }
-  return absl::StrFormat(
-      "      { \n"
-      "           %s"
-      "           %s"
-      "           %s"
-      "           \"action\": \"%s\"\n"
-      "      }",
-      path_match_str, absl::StrJoin(headers_service_config, ""),
-      route.matchers.fraction_per_million.has_value()
-          ? absl::StrFormat("\"match_fraction\":%d,\n",
-                            route.matchers.fraction_per_million.value())
-          : "",
-      action_name);
 }
 
-// Create the service config for one weighted cluster.
-std::string CreateServiceConfigActionWeightedCluster(
-    const std::string& name,
-    const std::vector<XdsApi::Route::ClusterWeight>& clusters) {
-  std::vector<std::string> config_parts;
-  config_parts.push_back(
-      absl::StrFormat("      \"weighted:%s\":{\n"
-                      "        \"childPolicy\":[ {\n"
-                      "          \"weighted_target_experimental\":{\n"
-                      "            \"targets\":{\n",
-                      name));
-  std::vector<std::string> weighted_targets;
-  weighted_targets.reserve(clusters.size());
-  for (const auto& cluster_weight : clusters) {
-    weighted_targets.push_back(absl::StrFormat(
-        "              \"%s\":{\n"
-        "                \"weight\":%d,\n"
-        "                \"childPolicy\":[ {\n"
-        "                  \"cds_experimental\":{\n"
-        "                    \"cluster\": \"%s\"\n"
-        "                  }\n"
-        "                } ]\n"
-        "               }",
-        cluster_weight.name, cluster_weight.weight, cluster_weight.name));
+void XdsResolver::OnRouteConfigUpdate(XdsApi::RdsUpdate rds_update) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) {
+    gpr_log(GPR_INFO, "[xds_resolver %p] received updated route config", this);
   }
-  config_parts.push_back(absl::StrJoin(weighted_targets, ",\n"));
-  config_parts.push_back(
-      "            }\n"
-      "          }\n"
-      "        } ]\n"
-      "       }");
-  return absl::StrJoin(config_parts, "");
-}
-
-struct WeightedClustersKeys {
-  std::string cluster_names_key;
-  std::string cluster_weights_key;
-};
-
-// Returns the cluster names and weights key or the cluster names only key.
-WeightedClustersKeys GetWeightedClustersKey(
-    const std::vector<XdsApi::Route::ClusterWeight>& weighted_clusters) {
-  std::set<std::string> cluster_names;
-  std::set<std::string> cluster_weights;
-  for (const auto& cluster_weight : weighted_clusters) {
-    cluster_names.emplace(absl::StrFormat("%s", cluster_weight.name));
-    cluster_weights.emplace(
-        absl::StrFormat("%s_%d", cluster_weight.name, cluster_weight.weight));
+  // Find the relevant VirtualHost from the RouteConfiguration.
+  XdsApi::RdsUpdate::VirtualHost* vhost =
+      rds_update.FindVirtualHostForDomain(server_name_);
+  if (vhost == nullptr) {
+    OnError(GRPC_ERROR_CREATE_FROM_COPIED_STRING(
+        absl::StrCat("could not find VirtualHost for ", server_name_,
+                     " in RouteConfiguration")
+            .c_str()));
+    return;
   }
-  return {absl::StrJoin(cluster_names, "_"),
-          absl::StrJoin(cluster_weights, "_")};
+  // Save the list of routes in the resolver.
+  current_update_ = std::move(vhost->routes);
+  // Send a new result to the channel.
+  GenerateResult();
 }
 
-std::string XdsResolver::WeightedClustersActionName(
-    const std::vector<XdsApi::Route::ClusterWeight>& weighted_clusters) {
-  WeightedClustersKeys keys = GetWeightedClustersKey(weighted_clusters);
-  auto cluster_names_map_it =
-      weighted_cluster_index_map_.find(keys.cluster_names_key);
-  GPR_ASSERT(cluster_names_map_it != weighted_cluster_index_map_.end());
-  const auto& cluster_weights_map =
-      cluster_names_map_it->second.cluster_weights_map;
-  auto cluster_weights_map_it =
-      cluster_weights_map.find(keys.cluster_weights_key);
-  GPR_ASSERT(cluster_weights_map_it != cluster_weights_map.end());
-  return absl::StrFormat("%s_%d", keys.cluster_names_key,
-                         cluster_weights_map_it->second);
+void XdsResolver::OnError(grpc_error* error) {
+  gpr_log(GPR_ERROR, "[xds_resolver %p] received error from XdsClient: %s",
+          this, grpc_error_string(error));
+  Result result;
+  result.args = grpc_channel_args_copy(args_);
+  result.service_config_error = error;
+  result_handler()->ReturnResult(std::move(result));
 }
 
-void XdsResolver::UpdateWeightedClusterIndexMap(
-    const std::vector<XdsApi::Route>& routes) {
-  // Construct a list of unique WeightedCluster
-  // actions which we need to process: to find action names
-  std::map<std::string /* cluster_weights_key */,
-           std::string /* cluster_names_key */>
-      actions_to_process;
-  for (const auto& route : routes) {
-    if (!route.weighted_clusters.empty()) {
-      WeightedClustersKeys keys =
-          GetWeightedClustersKey(route.weighted_clusters);
-      auto action_it = actions_to_process.find(keys.cluster_weights_key);
-      if (action_it == actions_to_process.end()) {
-        actions_to_process[std::move(keys.cluster_weights_key)] =
-            std::move(keys.cluster_names_key);
-      }
-    }
-  }
-  // First pass of all unique WeightedCluster actions: if the exact same
-  // weighted target policy (same clusters and weights) appears in the old map,
-  // then that old action name is taken again and should be moved to the new
-  // map; any other action names from the old set of actions are candidates for
-  // reuse.
-  XdsResolver::WeightedClusterIndexMap new_weighted_cluster_index_map;
-  for (auto action_it = actions_to_process.begin();
-       action_it != actions_to_process.end();) {
-    const std::string& cluster_names_key = action_it->second;
-    const std::string& cluster_weights_key = action_it->first;
-    auto old_cluster_names_map_it =
-        weighted_cluster_index_map_.find(cluster_names_key);
-    if (old_cluster_names_map_it != weighted_cluster_index_map_.end()) {
-      // Add cluster_names_key to the new map and copy next_index.
-      auto& new_cluster_names_info =
-          new_weighted_cluster_index_map[cluster_names_key];
-      new_cluster_names_info.next_index =
-          old_cluster_names_map_it->second.next_index;
-      // Lookup cluster_weights_key in old map.
-      auto& old_cluster_weights_map =
-          old_cluster_names_map_it->second.cluster_weights_map;
-      auto old_cluster_weights_map_it =
-          old_cluster_weights_map.find(cluster_weights_key);
-      if (old_cluster_weights_map_it != old_cluster_weights_map.end()) {
-        // same policy found, move from old map to new map.
-        new_cluster_names_info.cluster_weights_map[cluster_weights_key] =
-            old_cluster_weights_map_it->second;
-        old_cluster_weights_map.erase(old_cluster_weights_map_it);
-        // This action has been added to new map, so no need to process it
-        // again.
-        action_it = actions_to_process.erase(action_it);
-        continue;
-      }
-    }
-    ++action_it;
-  }
-  // Second pass of all remaining unique WeightedCluster actions: if clusters
-  // for a new action are the same as an old unused action, reuse the name.  If
-  // clusters differ, use a brand new name.
-  for (const auto& action : actions_to_process) {
-    const std::string& cluster_names_key = action.second;
-    const std::string& cluster_weights_key = action.first;
-    auto& new_cluster_names_info =
-        new_weighted_cluster_index_map[cluster_names_key];
-    auto& old_cluster_weights_map =
-        weighted_cluster_index_map_[cluster_names_key].cluster_weights_map;
-    auto old_cluster_weights_it = old_cluster_weights_map.begin();
-    if (old_cluster_weights_it != old_cluster_weights_map.end()) {
-      // There is something to reuse: this action uses the same set
-      // of clusters as a previous action and that action name is not
-      // already taken.
-      new_cluster_names_info.cluster_weights_map[cluster_weights_key] =
-          old_cluster_weights_it->second;
-      // Remove the name from being able to reuse again.
-      old_cluster_weights_map.erase(old_cluster_weights_it);
-    } else {
-      // There is nothing to reuse, take the next index to use and
-      // increment.
-      new_cluster_names_info.cluster_weights_map[cluster_weights_key] =
-          new_cluster_names_info.next_index++;
-    }
-  }
-  weighted_cluster_index_map_ = std::move(new_weighted_cluster_index_map);
+void XdsResolver::OnResourceDoesNotExist() {
+  gpr_log(GPR_ERROR,
+          "[xds_resolver %p] LDS/RDS resource does not exist -- clearing "
+          "update and returning empty service config",
+          this);
+  current_update_.clear();
+  Result result;
+  result.service_config =
+      ServiceConfig::Create(args_, "{}", &result.service_config_error);
+  GPR_ASSERT(result.service_config != nullptr);
+  result.args = grpc_channel_args_copy(args_);
+  result_handler()->ReturnResult(std::move(result));
 }
 
 grpc_error* XdsResolver::CreateServiceConfig(
-    const std::vector<XdsApi::Route>& routes,
     RefCountedPtr<ServiceConfig>* service_config) {
-  UpdateWeightedClusterIndexMap(routes);
-  std::vector<std::string> actions_vector;
-  std::vector<std::string> route_table;
-  std::set<std::string> actions_set;
-  for (const auto& route : routes) {
-    const std::string action_name =
-        route.weighted_clusters.empty()
-            ? route.cluster_name
-            : WeightedClustersActionName(route.weighted_clusters);
-    if (actions_set.find(action_name) == actions_set.end()) {
-      actions_set.emplace(action_name);
-      actions_vector.push_back(
-          route.weighted_clusters.empty()
-              ? CreateServiceConfigActionCluster(action_name)
-              : CreateServiceConfigActionWeightedCluster(
-                    action_name, route.weighted_clusters));
-    }
-    route_table.push_back(CreateServiceConfigRoute(
-        absl::StrFormat("%s:%s",
-                        route.weighted_clusters.empty() ? "cds" : "weighted",
-                        action_name),
-        route));
+  std::vector<std::string> clusters;
+  for (const auto& cluster : cluster_state_map_) {
+    clusters.push_back(
+        absl::StrFormat("      \"%s\":{\n"
+                        "        \"childPolicy\":[ {\n"
+                        "          \"cds_experimental\":{\n"
+                        "            \"cluster\": \"%s\"\n"
+                        "          }\n"
+                        "        } ]\n"
+                        "       }",
+                        cluster.first, cluster.first));
   }
   std::vector<std::string> config_parts;
   config_parts.push_back(
       "{\n"
       "  \"loadBalancingConfig\":[\n"
-      "    { \"xds_routing_experimental\":{\n"
-      "      \"actions\":{\n");
-  config_parts.push_back(absl::StrJoin(actions_vector, ",\n"));
-  config_parts.push_back(
-      "    },\n"
-      "      \"routes\":[\n");
-  config_parts.push_back(absl::StrJoin(route_table, ",\n"));
+      "    { \"xds_cluster_manager_experimental\":{\n"
+      "      \"children\":{\n");
+  config_parts.push_back(absl::StrJoin(clusters, ",\n"));
   config_parts.push_back(
-      "    ]\n"
+      "    }\n"
       "    } }\n"
       "  ]\n"
       "}");
   std::string json = absl::StrJoin(config_parts, "");
   grpc_error* error = GRPC_ERROR_NONE;
-  *service_config = ServiceConfig::Create(json.c_str(), &error);
+  *service_config = ServiceConfig::Create(args_, json.c_str(), &error);
   return error;
 }
 
+void XdsResolver::GenerateResult() {
+  if (current_update_.empty()) return;
+  // First create XdsConfigSelector, which may add new entries to the cluster
+  // state map, and then CreateServiceConfig for LB policies.
+  auto config_selector =
+      MakeRefCounted<XdsConfigSelector>(Ref(), current_update_);
+  Result result;
+  grpc_error* error = CreateServiceConfig(&result.service_config);
+  if (error != GRPC_ERROR_NONE) {
+    OnError(error);
+    return;
+  }
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) {
+    gpr_log(GPR_INFO, "[xds_resolver %p] generated service config: %s", this,
+            result.service_config->json_string().c_str());
+  }
+  grpc_arg new_arg = config_selector->MakeChannelArg();
+  result.args = grpc_channel_args_copy_and_add(args_, &new_arg, 1);
+  result_handler()->ReturnResult(std::move(result));
+}
+
+void XdsResolver::MaybeRemoveUnusedClusters() {
+  bool update_needed = false;
+  for (auto it = cluster_state_map_.begin(); it != cluster_state_map_.end();) {
+    RefCountedPtr<ClusterState> cluster_state = it->second->RefIfNonZero();
+    if (cluster_state != nullptr) {
+      ++it;
+    } else {
+      update_needed = true;
+      it = cluster_state_map_.erase(it);
+    }
+  }
+  if (update_needed && xds_client_ != nullptr) {
+    // Send a new result to the channel.
+    GenerateResult();
+  }
+}
+
 //
 // Factory
 //