grpc 1.48.0 → 1.49.1

data/src/core/ext/xds/xds_client.h

@@ -32,13 +32,11 @@
 #include "absl/strings/string_view.h"
 #include "upb/def.hpp"
 
-#include <grpc/impl/codegen/grpc_types.h>
-
-#include "src/core/ext/xds/certificate_provider_store.h"
 #include "src/core/ext/xds/xds_api.h"
 #include "src/core/ext/xds/xds_bootstrap.h"
 #include "src/core/ext/xds/xds_client_stats.h"
 #include "src/core/ext/xds/xds_resource_type.h"
+#include "src/core/ext/xds/xds_transport.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/dual_ref_counted.h"
 #include "src/core/lib/gprpp/orphanable.h"
@@ -46,10 +44,8 @@
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/gprpp/time.h"
-#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/gprpp/work_serializer.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/iomgr/iomgr_fwd.h"
-#include "src/core/lib/iomgr/work_serializer.h"
 #include "src/core/lib/uri/uri_parser.h"
 
 namespace grpc_core {
@@ -74,15 +70,9 @@ class XdsClient : public DualRefCounted<XdsClient> {
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) = 0;
   };
 
-  // Factory function to get or create the global XdsClient instance.
-  // If *error is not GRPC_ERROR_NONE upon return, then there was
-  // an error initializing the client.
-  static RefCountedPtr<XdsClient> GetOrCreate(const grpc_channel_args* args,
-                                              grpc_error_handle* error);
-
-  // Most callers should not instantiate directly.  Use GetOrCreate() instead.
   XdsClient(std::unique_ptr<XdsBootstrap> bootstrap,
-            const grpc_channel_args* args);
+            OrphanablePtr<XdsTransportFactory> transport_factory,
+            Duration resource_request_timeout = Duration::Seconds(15));
   ~XdsClient() override;
 
   const XdsBootstrap& bootstrap() const {
@@ -91,12 +81,10 @@ class XdsClient : public DualRefCounted<XdsClient> {
     return *bootstrap_;
   }
 
-  CertificateProviderStore& certificate_provider_store() {
-    return *certificate_provider_store_;
+  XdsTransportFactory* transport_factory() const {
+    return transport_factory_.get();
   }
 
-  grpc_pollset_set* interested_parties() const { return interested_parties_; }
-
   void Orphan() override;
 
   // Start and cancel watch for a resource.
@@ -158,11 +146,6 @@ class XdsClient : public DualRefCounted<XdsClient> {
   // implementation.
   std::string DumpClientConfigBinary();
 
-  // Helpers for encoding the XdsClient object in channel args.
-  grpc_arg MakeChannelArg() const;
-  static RefCountedPtr<XdsClient> GetFromChannelArgs(
-      const grpc_channel_args& args);
-
  private:
   struct XdsResourceKey {
     std::string id;
@@ -196,21 +179,18 @@ class XdsClient : public DualRefCounted<XdsClient> {
 
     void Orphan() override;
 
-    grpc_channel* channel() const { return channel_; }
     XdsClient* xds_client() const { return xds_client_.get(); }
     AdsCallState* ads_calld() const;
     LrsCallState* lrs_calld() const;
 
+    void ResetBackoff();
+
     void MaybeStartLrsCall();
     void StopLrsCallLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
     bool HasAdsCall() const;
     bool HasActiveAdsCall() const;
 
-    void StartConnectivityWatchLocked()
-        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
-    void CancelConnectivityWatchLocked();
-
     void SubscribeLocked(const XdsResourceType* type,
                          const XdsResourceName& name)
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
@@ -220,17 +200,18 @@ class XdsClient : public DualRefCounted<XdsClient> {
         ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
    private:
-    class StateWatcher;
+    void OnConnectivityStateChange(absl::Status status);
+    void OnConnectivityStateChangeLocked(absl::Status status)
+        ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_);
 
     // The owning xds client.
     WeakRefCountedPtr<XdsClient> xds_client_;
 
     const XdsBootstrap::XdsServer& server_;
 
-    // The channel and its status.
-    grpc_channel* channel_;
+    OrphanablePtr<XdsTransportFactory::XdsTransport> transport_;
+
     bool shutting_down_ = false;
-    StateWatcher* watcher_;
 
     // The retryable XDS calls.
     OrphanablePtr<RetryableCall<AdsCallState>> ads_calld_;
@@ -311,14 +292,13 @@ class XdsClient : public DualRefCounted<XdsClient> {
       const std::set<std::string>& clusters) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
 
   RefCountedPtr<ChannelState> GetOrCreateChannelStateLocked(
-      const XdsBootstrap::XdsServer& server) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+      const XdsBootstrap::XdsServer& server, const char* reason)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
 
   std::unique_ptr<XdsBootstrap> bootstrap_;
-  grpc_channel_args* args_;
+  OrphanablePtr<XdsTransportFactory> transport_factory_;
   const Duration request_timeout_;
   const bool xds_federation_enabled_;
-  grpc_pollset_set* interested_parties_;
-  OrphanablePtr<CertificateProviderStore> certificate_provider_store_;
   XdsApi api_;
   WorkSerializer work_serializer_;
 
@@ -349,14 +329,6 @@ class XdsClient : public DualRefCounted<XdsClient> {
   bool shutting_down_ ABSL_GUARDED_BY(mu_) = false;
 };
 
-namespace internal {
-void SetXdsChannelArgsForTest(grpc_channel_args* args);
-void UnsetGlobalXdsClientForTest();
-// Sets bootstrap config to be used when no env var is set.
-// Does not take ownership of config.
-void SetXdsFallbackBootstrapConfig(const char* config);
-}  // namespace internal
-
 }  // namespace grpc_core
 
 #endif  // GRPC_CORE_EXT_XDS_XDS_CLIENT_H

data/src/core/ext/xds/xds_client_grpc.cc

@@ -0,0 +1,291 @@
+//
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/ext/xds/xds_client_grpc.h"
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/memory/memory.h"
+#include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+
+#include <grpc/grpc.h>
+#include <grpc/slice.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+#include "src/core/ext/xds/certificate_provider_factory.h"
+#include "src/core/ext/xds/certificate_provider_registry.h"
+#include "src/core/ext/xds/xds_bootstrap.h"
+#include "src/core/ext/xds/xds_channel_args.h"
+#include "src/core/ext/xds/xds_cluster_specifier_plugin.h"
+#include "src/core/ext/xds/xds_http_filters.h"
+#include "src/core/ext/xds/xds_transport.h"
+#include "src/core/ext/xds/xds_transport_grpc.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/gprpp/time.h"
+#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/load_file.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/slice/slice_refcount.h"
+#include "src/core/lib/transport/error_utils.h"
+
+namespace grpc_core {
+
+//
+// GrpcXdsClient
+//
+
+namespace {
+
+Mutex* g_mu = nullptr;
+const grpc_channel_args* g_channel_args ABSL_GUARDED_BY(*g_mu) = nullptr;
+GrpcXdsClient* g_xds_client ABSL_GUARDED_BY(*g_mu) = nullptr;
+char* g_fallback_bootstrap_config ABSL_GUARDED_BY(*g_mu) = nullptr;
+
+}  // namespace
+
+void XdsClientGlobalInit() {
+  g_mu = new Mutex;
+  XdsHttpFilterRegistry::Init();
+  XdsClusterSpecifierPluginRegistry::Init();
+}
+
+// TODO(roth): Find a better way to clear the fallback config that does
+// not require using ABSL_NO_THREAD_SAFETY_ANALYSIS.
+void XdsClientGlobalShutdown() ABSL_NO_THREAD_SAFETY_ANALYSIS {
+  gpr_free(g_fallback_bootstrap_config);
+  g_fallback_bootstrap_config = nullptr;
+  delete g_mu;
+  g_mu = nullptr;
+  XdsHttpFilterRegistry::Shutdown();
+  XdsClusterSpecifierPluginRegistry::Shutdown();
+}
+
+namespace {
+
+absl::StatusOr<std::string> GetBootstrapContents(const char* fallback_config) {
+  // First, try GRPC_XDS_BOOTSTRAP env var.
+  UniquePtr<char> path(gpr_getenv("GRPC_XDS_BOOTSTRAP"));
+  if (path != nullptr) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+      gpr_log(GPR_INFO,
+              "Got bootstrap file location from GRPC_XDS_BOOTSTRAP "
+              "environment variable: %s",
+              path.get());
+    }
+    grpc_slice contents;
+    grpc_error_handle error =
+        grpc_load_file(path.get(), /*add_null_terminator=*/true, &contents);
+    if (!GRPC_ERROR_IS_NONE(error)) return grpc_error_to_absl_status(error);
+    std::string contents_str(StringViewFromSlice(contents));
+    grpc_slice_unref_internal(contents);
+    return contents_str;
+  }
+  // Next, try GRPC_XDS_BOOTSTRAP_CONFIG env var.
+  UniquePtr<char> env_config(gpr_getenv("GRPC_XDS_BOOTSTRAP_CONFIG"));
+  if (env_config != nullptr) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+      gpr_log(GPR_INFO,
+              "Got bootstrap contents from GRPC_XDS_BOOTSTRAP_CONFIG "
+              "environment variable");
+    }
+    return env_config.get();
+  }
+  // Finally, try fallback config.
+  if (fallback_config != nullptr) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+      gpr_log(GPR_INFO, "Got bootstrap contents from fallback config");
+    }
+    return fallback_config;
+  }
+  // No bootstrap config found.
+  return absl::FailedPreconditionError(
+      "Environment variables GRPC_XDS_BOOTSTRAP or GRPC_XDS_BOOTSTRAP_CONFIG "
+      "not defined");
+}
+
+}  // namespace
+
+absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GrpcXdsClient::GetOrCreate(
+    const ChannelArgs& args, const char* reason) {
+  // Construct certificate provider plugin map.
+  auto certificate_provider_plugin_map =
+      absl::make_unique<GrpcXdsCertificateProviderPluginMap>();
+  // If getting bootstrap from channel args, create a local XdsClient
+  // instance for the channel or server instead of using the global instance.
+  absl::optional<absl::string_view> bootstrap_config = args.GetString(
+      GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG);
+  if (bootstrap_config.has_value()) {
+    grpc_error_handle error = GRPC_ERROR_NONE;
+    std::unique_ptr<XdsBootstrap> bootstrap = XdsBootstrap::Create(
+        *bootstrap_config, std::move(certificate_provider_plugin_map), &error);
+    if (!GRPC_ERROR_IS_NONE(error)) return grpc_error_to_absl_status(error);
+    grpc_channel_args* xds_channel_args = args.GetPointer<grpc_channel_args>(
+        GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS);
+    return MakeRefCounted<GrpcXdsClient>(std::move(bootstrap),
+                                         ChannelArgs::FromC(xds_channel_args));
+  }
+  // Otherwise, use the global instance.
+  MutexLock lock(g_mu);
+  if (g_xds_client != nullptr) {
+    auto xds_client = g_xds_client->RefIfNonZero(DEBUG_LOCATION, reason);
+    if (xds_client != nullptr) return xds_client;
+  }
+  // Find bootstrap contents.
+  auto bootstrap_contents = GetBootstrapContents(g_fallback_bootstrap_config);
+  if (!bootstrap_contents.ok()) return bootstrap_contents.status();
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
+    gpr_log(GPR_INFO, "xDS bootstrap contents: %s",
+            bootstrap_contents->c_str());
+  }
+  // Parse bootstrap.
+  grpc_error_handle error = GRPC_ERROR_NONE;
+  std::unique_ptr<XdsBootstrap> bootstrap = XdsBootstrap::Create(
+      *bootstrap_contents, std::move(certificate_provider_plugin_map), &error);
+  if (!GRPC_ERROR_IS_NONE(error)) return grpc_error_to_absl_status(error);
+  // Instantiate XdsClient.
+  auto xds_client = MakeRefCounted<GrpcXdsClient>(
+      std::move(bootstrap), ChannelArgs::FromC(g_channel_args));
+  g_xds_client = xds_client.get();
+  return xds_client;
+}
+
+GrpcXdsClient::GrpcXdsClient(std::unique_ptr<XdsBootstrap> bootstrap,
+                             const ChannelArgs& args)
+    : XdsClient(
+          std::move(bootstrap), MakeOrphanable<GrpcXdsTransportFactory>(args),
+          std::max(Duration::Zero(),
+                   args.GetDurationFromIntMillis(
+                           GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS)
+                       .value_or(Duration::Seconds(15)))),
+      certificate_provider_store_(MakeOrphanable<CertificateProviderStore>(
+          static_cast<const GrpcXdsCertificateProviderPluginMap*>(
+              this->bootstrap().certificate_provider_plugin_map())
+              ->plugin_map())) {}
+
+GrpcXdsClient::~GrpcXdsClient() {
+  MutexLock lock(g_mu);
+  if (g_xds_client == this) g_xds_client = nullptr;
+}
+
+grpc_pollset_set* GrpcXdsClient::interested_parties() const {
+  return reinterpret_cast<GrpcXdsTransportFactory*>(transport_factory())
+      ->interested_parties();
+}
+
+namespace internal {
+
+void SetXdsChannelArgsForTest(grpc_channel_args* args) {
+  MutexLock lock(g_mu);
+  g_channel_args = args;
+}
+
+void UnsetGlobalXdsClientForTest() {
+  MutexLock lock(g_mu);
+  g_xds_client = nullptr;
+}
+
+void SetXdsFallbackBootstrapConfig(const char* config) {
+  MutexLock lock(g_mu);
+  gpr_free(g_fallback_bootstrap_config);
+  g_fallback_bootstrap_config = gpr_strdup(config);
+}
+
+}  // namespace internal
+
+//
+// GrpcXdsCertificateProviderPluginMap
+//
+
+absl::Status GrpcXdsCertificateProviderPluginMap::AddPlugin(
+    const std::string& instance_name, const std::string& plugin_name,
+    const Json& config) {
+  CertificateProviderFactory* factory =
+      CertificateProviderRegistry::LookupCertificateProviderFactory(
+          plugin_name);
+  if (factory == nullptr) {
+    return absl::InvalidArgumentError(
+        absl::StrCat("Unrecognized plugin name: ", plugin_name));
+  }
+  grpc_error_handle error = GRPC_ERROR_NONE;
+  auto parsed_config = factory->CreateCertificateProviderConfig(config, &error);
+  if (!GRPC_ERROR_IS_NONE(error)) {
+    absl::Status status = grpc_error_to_absl_status(error);
+    GRPC_ERROR_UNREF(error);
+    return status;
+  }
+  plugin_map_.insert({instance_name, {plugin_name, std::move(parsed_config)}});
+  return absl::OkStatus();
+}
+
+bool GrpcXdsCertificateProviderPluginMap::HasPlugin(
+    const std::string& instance_name) const {
+  return plugin_map_.find(instance_name) != plugin_map_.end();
+}
+
+std::string GrpcXdsCertificateProviderPluginMap::ToString() const {
+  std::vector<std::string> parts = {"{\n"};
+  for (const auto& entry : plugin_map_) {
+    parts.push_back(
+        absl::StrFormat("  %s={\n"
+                        "    plugin_name=%s\n"
+                        "    config=%s\n"
+                        "  },\n",
+                        entry.first, entry.second.plugin_name,
+                        entry.second.config->ToString()));
+  }
+  parts.push_back("}");
+  return absl::StrJoin(parts, "");
+}
+
+}  // namespace grpc_core
+
+// The returned bytes may contain NULL(0), so we can't use c-string.
+grpc_slice grpc_dump_xds_configs(void) {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
+  grpc_core::ExecCtx exec_ctx;
+  grpc_error_handle error = GRPC_ERROR_NONE;
+  auto xds_client = grpc_core::GrpcXdsClient::GetOrCreate(
+      grpc_core::ChannelArgs(), "grpc_dump_xds_configs()");
+  if (!xds_client.ok()) {
+    // If we aren't using xDS, just return an empty string.
+    return grpc_empty_slice();
+  }
+  return grpc_slice_from_cpp_string((*xds_client)->DumpClientConfigBinary());
+}

data/src/core/ext/xds/xds_client_grpc.h

@@ -0,0 +1,102 @@
+//
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef GRPC_CORE_EXT_XDS_XDS_CLIENT_GRPC_H
+#define GRPC_CORE_EXT_XDS_XDS_CLIENT_GRPC_H
+
+#include <grpc/support/port_platform.h>
+
+#include <memory>
+#include <string>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "absl/strings/string_view.h"
+
+#include <grpc/impl/codegen/grpc_types.h>
+
+#include "src/core/ext/xds/certificate_provider_store.h"
+#include "src/core/ext/xds/xds_bootstrap.h"
+#include "src/core/ext/xds/xds_client.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/iomgr/iomgr_fwd.h"
+#include "src/core/lib/json/json.h"
+
+namespace grpc_core {
+
+class GrpcXdsClient : public XdsClient {
+ public:
+  // Factory function to get or create the global XdsClient instance.
+  static absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GetOrCreate(
+      const ChannelArgs& args, const char* reason);
+
+  // Do not instantiate directly -- use GetOrCreate() instead.
+  GrpcXdsClient(std::unique_ptr<XdsBootstrap> bootstrap,
+                const ChannelArgs& args);
+  ~GrpcXdsClient() override;
+
+  // Helpers for encoding the XdsClient object in channel args.
+  static absl::string_view ChannelArgName() {
+    return "grpc.internal.xds_client";
+  }
+  static int ChannelArgsCompare(const XdsClient* a, const XdsClient* b) {
+    return QsortCompare(a, b);
+  }
+
+  grpc_pollset_set* interested_parties() const;
+
+  CertificateProviderStore& certificate_provider_store() const {
+    return *certificate_provider_store_;
+  }
+
+ private:
+  OrphanablePtr<CertificateProviderStore> certificate_provider_store_;
+};
+
+namespace internal {
+void SetXdsChannelArgsForTest(grpc_channel_args* args);
+void UnsetGlobalXdsClientForTest();
+// Sets bootstrap config to be used when no env var is set.
+// Does not take ownership of config.
+void SetXdsFallbackBootstrapConfig(const char* config);
+}  // namespace internal
+
+// Exposed for testing purposes only.
+class GrpcXdsCertificateProviderPluginMap
+    : public XdsCertificateProviderPluginMapInterface {
+ public:
+  const CertificateProviderStore::PluginDefinitionMap& plugin_map() const {
+    return plugin_map_;
+  }
+
+  absl::Status AddPlugin(const std::string& instance_name,
+                         const std::string& plugin_name,
+                         const Json& config) override;
+
+  bool HasPlugin(const std::string& instance_name) const override;
+
+  std::string ToString() const override;
+
+ private:
+  CertificateProviderStore::PluginDefinitionMap plugin_map_;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_EXT_XDS_XDS_CLIENT_GRPC_H