grpc 1.36.0 → 1.37.0.pre1

data/src/core/ext/xds/xds_bootstrap.h

@@ -54,7 +54,7 @@ class XdsBootstrap {
     std::string cluster;
     std::string locality_region;
     std::string locality_zone;
-    std::string locality_subzone;
+    std::string locality_sub_zone;
     Json metadata;
   };
 
@@ -88,6 +88,9 @@ class XdsBootstrap {
   // add support for fallback for the xds channel.
   const XdsServer& server() const { return servers_[0]; }
   const Node* node() const { return node_.get(); }
+  const std::string& server_listener_resource_name_template() const {
+    return server_listener_resource_name_template_;
+  }
 
   const CertificateProviderStore::PluginDefinitionMap& certificate_providers()
       const {
@@ -108,6 +111,7 @@ class XdsBootstrap {
 
   absl::InlinedVector<XdsServer, 1> servers_;
   std::unique_ptr<Node> node_;
+  std::string server_listener_resource_name_template_;
   CertificateProviderStore::PluginDefinitionMap certificate_providers_;
 };
 

data/src/core/ext/xds/xds_client.cc

@@ -35,9 +35,11 @@
 #include "src/core/ext/filters/client_channel/client_channel.h"
 #include "src/core/ext/filters/client_channel/service_config.h"
 #include "src/core/ext/xds/xds_api.h"
+#include "src/core/ext/xds/xds_bootstrap.h"
 #include "src/core/ext/xds/xds_channel_args.h"
 #include "src/core/ext/xds/xds_client.h"
 #include "src/core/ext/xds/xds_client_stats.h"
+#include "src/core/ext/xds/xds_http_filters.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
@@ -194,28 +196,34 @@ class XdsClient::ChannelState::AdsCallState
                 "timeout obtaining resource {type=%s name=%s} from xds server",
                 type_url_, name_)
                 .c_str());
+        watcher_error = grpc_error_set_int(
+            watcher_error, GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
         if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
           gpr_log(GPR_INFO, "[xds_client %p] %s", ads_calld_->xds_client(),
                   grpc_error_string(watcher_error));
         }
         if (type_url_ == XdsApi::kLdsTypeUrl) {
           ListenerState& state = ads_calld_->xds_client()->listener_map_[name_];
+          state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST;
           for (const auto& p : state.watchers) {
             p.first->OnError(GRPC_ERROR_REF(watcher_error));
           }
         } else if (type_url_ == XdsApi::kRdsTypeUrl) {
           RouteConfigState& state =
               ads_calld_->xds_client()->route_config_map_[name_];
+          state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST;
           for (const auto& p : state.watchers) {
             p.first->OnError(GRPC_ERROR_REF(watcher_error));
           }
         } else if (type_url_ == XdsApi::kCdsTypeUrl) {
           ClusterState& state = ads_calld_->xds_client()->cluster_map_[name_];
+          state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST;
           for (const auto& p : state.watchers) {
             p.first->OnError(GRPC_ERROR_REF(watcher_error));
           }
         } else if (type_url_ == XdsApi::kEdsTypeUrl) {
           EndpointState& state = ads_calld_->xds_client()->endpoint_map_[name_];
+          state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST;
           for (const auto& p : state.watchers) {
             p.first->OnError(GRPC_ERROR_REF(watcher_error));
           }
@@ -251,10 +259,14 @@ class XdsClient::ChannelState::AdsCallState
 
   void SendMessageLocked(const std::string& type_url);
 
-  void AcceptLdsUpdate(XdsApi::LdsUpdateMap lds_update_map);
-  void AcceptRdsUpdate(XdsApi::RdsUpdateMap rds_update_map);
-  void AcceptCdsUpdate(XdsApi::CdsUpdateMap cds_update_map);
-  void AcceptEdsUpdate(XdsApi::EdsUpdateMap eds_update_map);
+  void AcceptLdsUpdate(std::string version, grpc_millis update_time,
+                       XdsApi::LdsUpdateMap lds_update_map);
+  void AcceptRdsUpdate(std::string version, grpc_millis update_time,
+                       XdsApi::RdsUpdateMap rds_update_map);
+  void AcceptCdsUpdate(std::string version, grpc_millis update_time,
+                       XdsApi::CdsUpdateMap cds_update_map);
+  void AcceptEdsUpdate(std::string version, grpc_millis update_time,
+                       XdsApi::EdsUpdateMap eds_update_map);
 
   static void OnRequestSent(void* arg, grpc_error* error);
   void OnRequestSentLocked(grpc_error* error);
@@ -503,7 +515,7 @@ XdsClient::ChannelState::LrsCallState* XdsClient::ChannelState::lrs_calld()
 }
 
 bool XdsClient::ChannelState::HasActiveAdsCall() const {
-  return ads_calld_->calld() != nullptr;
+  return ads_calld_ != nullptr && ads_calld_->calld() != nullptr;
 }
 
 void XdsClient::ChannelState::MaybeStartLrsCall() {
@@ -866,7 +878,24 @@ bool XdsClient::ChannelState::AdsCallState::HasSubscribedResources() const {
   return false;
 }
 
+namespace {
+
+// Build a resource metadata struct for ADS result accepting methods and CSDS.
+XdsApi::ResourceMetadata CreateResourceMetadataAcked(
+    std::string serialized_proto, std::string version,
+    grpc_millis update_time) {
+  XdsApi::ResourceMetadata resource_metadata;
+  resource_metadata.serialized_proto = std::move(serialized_proto);
+  resource_metadata.update_time = update_time;
+  resource_metadata.version = std::move(version);
+  resource_metadata.client_status = XdsApi::ResourceMetadata::ACKED;
+  return resource_metadata;
+}
+
+}  // namespace
+
 void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate(
+    std::string version, grpc_millis update_time,
     XdsApi::LdsUpdateMap lds_update_map) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO,
@@ -878,7 +907,7 @@ void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate(
   std::set<std::string> rds_resource_names_seen;
   for (auto& p : lds_update_map) {
     const std::string& listener_name = p.first;
-    XdsApi::LdsUpdate& lds_update = p.second;
+    XdsApi::LdsUpdate& lds_update = p.second.resource;
     auto& state = lds_state.subscribed_resources[listener_name];
     if (state != nullptr) state->Finish();
     if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
@@ -886,8 +915,9 @@ void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate(
               listener_name.c_str(), lds_update.ToString().c_str());
     }
     // Record the RDS resource names seen.
-    if (!lds_update.route_config_name.empty()) {
-      rds_resource_names_seen.insert(lds_update.route_config_name);
+    if (!lds_update.http_connection_manager.route_config_name.empty()) {
+      rds_resource_names_seen.insert(
+          lds_update.http_connection_manager.route_config_name);
     }
     // Ignore identical update.
     ListenerState& listener_state = xds_client()->listener_map_[listener_name];
@@ -903,6 +933,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate(
     }
     // Update the listener state.
     listener_state.update = std::move(lds_update);
+    listener_state.meta = CreateResourceMetadataAcked(
+        std::move(p.second.serialized_proto), version, update_time);
     // Notify watchers.
     for (const auto& p : listener_state.watchers) {
       p.first->OnListenerChanged(*listener_state.update);
@@ -947,6 +979,7 @@ void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate(
 }
 
 void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdate(
+    std::string version, grpc_millis update_time,
     XdsApi::RdsUpdateMap rds_update_map) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO,
@@ -957,7 +990,7 @@ void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdate(
   auto& rds_state = state_map_[XdsApi::kRdsTypeUrl];
   for (auto& p : rds_update_map) {
     const std::string& route_config_name = p.first;
-    XdsApi::RdsUpdate& rds_update = p.second;
+    XdsApi::RdsUpdate& rds_update = p.second.resource;
     auto& state = rds_state.subscribed_resources[route_config_name];
     if (state != nullptr) state->Finish();
     if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
@@ -978,6 +1011,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdate(
     }
     // Update the cache.
     route_config_state.update = std::move(rds_update);
+    route_config_state.meta = CreateResourceMetadataAcked(
+        std::move(p.second.serialized_proto), version, update_time);
     // Notify all watchers.
     for (const auto& p : route_config_state.watchers) {
       p.first->OnRouteConfigChanged(*route_config_state.update);
@@ -986,6 +1021,7 @@ void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdate(
 }
 
 void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdate(
+    std::string version, grpc_millis update_time,
     XdsApi::CdsUpdateMap cds_update_map) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO,
@@ -997,7 +1033,7 @@ void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdate(
   std::set<std::string> eds_resource_names_seen;
   for (auto& p : cds_update_map) {
     const char* cluster_name = p.first.c_str();
-    XdsApi::CdsUpdate& cds_update = p.second;
+    XdsApi::CdsUpdate& cds_update = p.second.resource;
     auto& state = cds_state.subscribed_resources[cluster_name];
     if (state != nullptr) state->Finish();
     if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
@@ -1021,6 +1057,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdate(
     }
     // Update the cluster state.
     cluster_state.update = std::move(cds_update);
+    cluster_state.meta = CreateResourceMetadataAcked(
+        std::move(p.second.serialized_proto), version, update_time);
     // Notify all watchers.
     for (const auto& p : cluster_state.watchers) {
       p.first->OnClusterChanged(cluster_state.update.value());
@@ -1064,6 +1102,7 @@ void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdate(
 }
 
 void XdsClient::ChannelState::AdsCallState::AcceptEdsUpdate(
+    std::string version, grpc_millis update_time,
     XdsApi::EdsUpdateMap eds_update_map) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
     gpr_log(GPR_INFO,
@@ -1074,7 +1113,7 @@ void XdsClient::ChannelState::AdsCallState::AcceptEdsUpdate(
   auto& eds_state = state_map_[XdsApi::kEdsTypeUrl];
   for (auto& p : eds_update_map) {
     const char* eds_service_name = p.first.c_str();
-    XdsApi::EdsUpdate& eds_update = p.second;
+    XdsApi::EdsUpdate& eds_update = p.second.resource;
     auto& state = eds_state.subscribed_resources[eds_service_name];
     if (state != nullptr) state->Finish();
     if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
@@ -1095,6 +1134,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptEdsUpdate(
     }
     // Update the cluster state.
     endpoint_state.update = std::move(eds_update);
+    endpoint_state.meta = CreateResourceMetadataAcked(
+        std::move(p.second.serialized_proto), version, update_time);
     // Notify all watchers.
     for (const auto& p : endpoint_state.watchers) {
       p.first->OnEndpointChanged(endpoint_state.update.value());
@@ -1161,7 +1202,8 @@ bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() {
   recv_message_payload_ = nullptr;
   // Parse and validate the response.
   XdsApi::AdsParseResult result = xds_client()->api_.ParseAdsResponse(
-      response_slice, ResourceNamesForRequest(XdsApi::kLdsTypeUrl),
+      chand()->server_, response_slice,
+      ResourceNamesForRequest(XdsApi::kLdsTypeUrl),
       ResourceNamesForRequest(XdsApi::kRdsTypeUrl),
       ResourceNamesForRequest(XdsApi::kCdsTypeUrl),
       ResourceNamesForRequest(XdsApi::kEdsTypeUrl));
@@ -1173,11 +1215,14 @@ bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() {
             xds_client(), grpc_error_string(result.parse_error));
     GRPC_ERROR_UNREF(result.parse_error);
   } else {
+    grpc_millis update_time = grpc_core::ExecCtx::Get()->Now();
     // Update nonce.
     auto& state = state_map_[result.type_url];
     state.nonce = std::move(result.nonce);
     // NACK or ACK the response.
     if (result.parse_error != GRPC_ERROR_NONE) {
+      xds_client()->UpdateResourceMetadataWithFailedParseResult(update_time,
+                                                                result);
       GRPC_ERROR_UNREF(state.error);
       state.error = result.parse_error;
       // NACK unacceptable update.
@@ -1191,13 +1236,17 @@ bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() {
       seen_response_ = true;
       // Accept the ADS response according to the type_url.
       if (result.type_url == XdsApi::kLdsTypeUrl) {
-        AcceptLdsUpdate(std::move(result.lds_update_map));
+        AcceptLdsUpdate(result.version, update_time,
+                        std::move(result.lds_update_map));
       } else if (result.type_url == XdsApi::kRdsTypeUrl) {
-        AcceptRdsUpdate(std::move(result.rds_update_map));
+        AcceptRdsUpdate(result.version, update_time,
+                        std::move(result.rds_update_map));
       } else if (result.type_url == XdsApi::kCdsTypeUrl) {
-        AcceptCdsUpdate(std::move(result.cds_update_map));
+        AcceptCdsUpdate(result.version, update_time,
+                        std::move(result.cds_update_map));
       } else if (result.type_url == XdsApi::kEdsTypeUrl) {
-        AcceptEdsUpdate(std::move(result.eds_update_map));
+        AcceptEdsUpdate(result.version, update_time,
+                        std::move(result.eds_update_map));
       }
       xds_client()->resource_version_map_[result.type_url] =
           std::move(result.version);
@@ -2192,27 +2241,109 @@ XdsApi::ClusterLoadReportMap XdsClient::BuildLoadReportSnapshotLocked(
   return snapshot_map;
 }
 
+void XdsClient::UpdateResourceMetadataWithFailedParseResult(
+    grpc_millis update_time, const XdsApi::AdsParseResult& result) {
+  // ADS update is rejected and the resource names in the failed update is
+  // available.
+  absl::string_view details = grpc_error_string(result.parse_error);
+  for (auto& name : result.resource_names_failed) {
+    XdsApi::ResourceMetadata* resource_metadata = nullptr;
+    if (result.type_url == XdsApi::kLdsTypeUrl) {
+      auto it = listener_map_.find(name);
+      if (it != listener_map_.end()) {
+        resource_metadata = &it->second.meta;
+      }
+    } else if (result.type_url == XdsApi::kRdsTypeUrl) {
+      auto it = route_config_map_.find(name);
+      if (route_config_map_.find(name) != route_config_map_.end()) {
+        resource_metadata = &it->second.meta;
+      }
+    } else if (result.type_url == XdsApi::kCdsTypeUrl) {
+      auto it = cluster_map_.find(name);
+      if (cluster_map_.find(name) != cluster_map_.end()) {
+        resource_metadata = &it->second.meta;
+      }
+    } else if (result.type_url == XdsApi::kEdsTypeUrl) {
+      auto it = endpoint_map_.find(name);
+      if (endpoint_map_.find(name) != endpoint_map_.end()) {
+        resource_metadata = &it->second.meta;
+      }
+    }
+    if (resource_metadata == nullptr) {
+      return;
+    }
+    resource_metadata->client_status = XdsApi::ResourceMetadata::NACKED;
+    resource_metadata->failed_version = result.version;
+    resource_metadata->failed_details = std::string(details);
+    resource_metadata->failed_update_time = update_time;
+  }
+}
+
+std::string XdsClient::DumpClientConfigBinary() {
+  MutexLock lock(&mu_);
+  XdsApi::ResourceTypeMetadataMap resource_type_metadata_map;
+  // Update per-xds-type version if available, this version corresponding to the
+  // last successful ADS update version.
+  for (auto& p : resource_version_map_) {
+    resource_type_metadata_map[p.first].version = p.second;
+  }
+  // Collect resource metadata from listeners
+  auto& lds_map =
+      resource_type_metadata_map[XdsApi::kLdsTypeUrl].resource_metadata_map;
+  for (auto& p : listener_map_) {
+    lds_map[p.first] = &p.second.meta;
+  }
+  // Collect resource metadata from route configs
+  auto& rds_map =
+      resource_type_metadata_map[XdsApi::kRdsTypeUrl].resource_metadata_map;
+  for (auto& p : route_config_map_) {
+    rds_map[p.first] = &p.second.meta;
+  }
+  // Collect resource metadata from clusters
+  auto& cds_map =
+      resource_type_metadata_map[XdsApi::kCdsTypeUrl].resource_metadata_map;
+  for (auto& p : cluster_map_) {
+    cds_map[p.first] = &p.second.meta;
+  }
+  // Collect resource metadata from endpoints
+  auto& eds_map =
+      resource_type_metadata_map[XdsApi::kEdsTypeUrl].resource_metadata_map;
+  for (auto& p : endpoint_map_) {
+    eds_map[p.first] = &p.second.meta;
+  }
+  // Assemble config dump messages
+  return api_.AssembleClientConfig(resource_type_metadata_map);
+}
+
 //
 // accessors for global state
 //
 
-void XdsClientGlobalInit() { g_mu = new Mutex; }
+void XdsClientGlobalInit() {
+  g_mu = new Mutex;
+  XdsHttpFilterRegistry::Init();
+}
 
 void XdsClientGlobalShutdown() {
   delete g_mu;
   g_mu = nullptr;
   gpr_free(g_fallback_bootstrap_config);
   g_fallback_bootstrap_config = nullptr;
+  XdsHttpFilterRegistry::Shutdown();
 }
 
 RefCountedPtr<XdsClient> XdsClient::GetOrCreate(grpc_error** error) {
-  MutexLock lock(g_mu);
-  if (g_xds_client != nullptr) {
-    auto xds_client = g_xds_client->RefIfNonZero();
-    if (xds_client != nullptr) return xds_client;
+  RefCountedPtr<XdsClient> xds_client;
+  {
+    MutexLock lock(g_mu);
+    if (g_xds_client != nullptr) {
+      auto xds_client = g_xds_client->RefIfNonZero();
+      if (xds_client != nullptr) return xds_client;
+    }
+    xds_client = MakeRefCounted<XdsClient>(error);
+    if (*error != GRPC_ERROR_NONE) return nullptr;
+    g_xds_client = xds_client.get();
   }
-  auto xds_client = MakeRefCounted<XdsClient>(error);
-  g_xds_client = xds_client.get();
   return xds_client;
 }
 
@@ -2237,3 +2368,17 @@ void SetXdsFallbackBootstrapConfig(const char* config) {
 }  // namespace internal
 
 }  // namespace grpc_core
+
+// The returned bytes may contain NULL(0), so we can't use c-string.
+grpc_slice grpc_dump_xds_configs() {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
+  grpc_core::ExecCtx exec_ctx;
+  grpc_error* error = GRPC_ERROR_NONE;
+  auto xds_client = grpc_core::XdsClient::GetOrCreate(&error);
+  if (error != GRPC_ERROR_NONE) {
+    // If we isn't using xDS, just return an empty string.
+    GRPC_ERROR_UNREF(error);
+    return grpc_empty_slice();
+  }
+  return grpc_slice_from_cpp_string(xds_client->DumpClientConfigBinary());
+}

data/src/core/ext/xds/xds_client.h

@@ -88,6 +88,12 @@ class XdsClient : public DualRefCounted<XdsClient> {
   explicit XdsClient(grpc_error** error);
   ~XdsClient() override;
 
+  const XdsBootstrap& bootstrap() const {
+    // bootstrap_ is guaranteed to be non-null since XdsClient::GetOrCreate()
+    // would return a null object if bootstrap_ was null.
+    return *bootstrap_;
+  }
+
   CertificateProviderStore& certificate_provider_store() {
     return *certificate_provider_store_;
   }
@@ -185,6 +191,15 @@ class XdsClient : public DualRefCounted<XdsClient> {
   // Resets connection backoff state.
   void ResetBackoff();
 
+  // Dumps the active xDS config in JSON format.
+  // Individual xDS resource is encoded as envoy.admin.v3.*ConfigDump. Returns
+  // envoy.service.status.v3.ClientConfig which also includes the config
+  // status (e.g., CLIENT_REQUESTED, CLIENT_ACKED, CLIENT_NACKED).
+  //
+  // Expected to be invoked by wrapper languages in their CSDS service
+  // implementation.
+  std::string DumpClientConfigBinary();
+
  private:
   // Contains a channel to the xds server and all the data related to the
   // channel.  Holds a ref to the xds client object.
@@ -215,6 +230,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
     void MaybeStartLrsCall();
     void StopLrsCall();
 
+    bool HasAdsCall() const;
     bool HasActiveAdsCall() const;
 
     void StartConnectivityWatchLocked();
@@ -248,6 +264,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
         watchers;
     // The latest data seen from LDS.
     absl::optional<XdsApi::LdsUpdate> update;
+    XdsApi::ResourceMetadata meta;
   };
 
   struct RouteConfigState {
@@ -256,6 +273,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
         watchers;
     // The latest data seen from RDS.
     absl::optional<XdsApi::RdsUpdate> update;
+    XdsApi::ResourceMetadata meta;
   };
 
   struct ClusterState {
@@ -263,6 +281,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
         watchers;
     // The latest data seen from CDS.
     absl::optional<XdsApi::CdsUpdate> update;
+    XdsApi::ResourceMetadata meta;
   };
 
   struct EndpointState {
@@ -271,6 +290,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
         watchers;
     // The latest data seen from EDS.
     absl::optional<XdsApi::EdsUpdate> update;
+    XdsApi::ResourceMetadata meta;
   };
 
   struct LoadReportState {
@@ -293,6 +313,12 @@ class XdsClient : public DualRefCounted<XdsClient> {
   XdsApi::ClusterLoadReportMap BuildLoadReportSnapshotLocked(
       bool send_all_clusters, const std::set<std::string>& clusters);
 
+  void UpdateResourceMetadataWithFailedParseResult(
+      grpc_millis update_time, const XdsApi::AdsParseResult& result);
+  void UpdatePendingResources(
+      const std::string& type_url,
+      XdsApi::ResourceMetadataMap* resource_metadata_map);
+
   const grpc_millis request_timeout_;
   grpc_pollset_set* interested_parties_;
   std::unique_ptr<XdsBootstrap> bootstrap_;