grpc 1.67.0.pre1 → 1.68.1

data/src/core/xds/xds_client/xds_api.cc

@@ -16,6 +16,9 @@
 
 #include "src/core/xds/xds_client/xds_api.h"
 
+#include <grpc/status.h>
+#include <grpc/support/port_platform.h>
+#include <grpc/support/time.h>
 #include <stdint.h>
 #include <stdlib.h>
 
@@ -38,19 +41,14 @@
 #include "google/protobuf/struct.upb.h"
 #include "google/protobuf/timestamp.upb.h"
 #include "google/rpc/status.upb.h"
+#include "src/core/util/json/json.h"
+#include "src/core/util/upb_utils.h"
+#include "src/core/xds/xds_client/xds_client.h"
 #include "upb/base/string_view.h"
 #include "upb/mem/arena.hpp"
 #include "upb/reflection/def.h"
 #include "upb/text/encode.h"
 
-#include <grpc/status.h>
-#include <grpc/support/port_platform.h>
-#include <grpc/support/time.h>
-
-#include "src/core/util/json/json.h"
-#include "src/core/util/upb_utils.h"
-#include "src/core/xds/xds_client/xds_client.h"
-
 // IWYU pragma: no_include "upb/msg_internal.h"
 
 namespace grpc_core {
@@ -154,42 +152,50 @@ std::string SerializeDiscoveryRequest(
 
 void XdsApi::PopulateNode(envoy_config_core_v3_Node* node_msg,
                           upb_Arena* arena) {
-  if (node_ != nullptr) {
-    if (!node_->id().empty()) {
+  PopulateXdsNode(node_, user_agent_name_, user_agent_version_, node_msg,
+                  arena);
+}
+
+void PopulateXdsNode(const XdsBootstrap::Node* node,
+                     absl::string_view user_agent_name,
+                     absl::string_view user_agent_version,
+                     envoy_config_core_v3_Node* node_msg, upb_Arena* arena) {
+  if (node != nullptr) {
+    if (!node->id().empty()) {
       envoy_config_core_v3_Node_set_id(node_msg,
-                                       StdStringToUpbString(node_->id()));
+                                       StdStringToUpbString(node->id()));
     }
-    if (!node_->cluster().empty()) {
+    if (!node->cluster().empty()) {
       envoy_config_core_v3_Node_set_cluster(
-          node_msg, StdStringToUpbString(node_->cluster()));
+          node_msg, StdStringToUpbString(node->cluster()));
     }
-    if (!node_->metadata().empty()) {
+    if (!node->metadata().empty()) {
       google_protobuf_Struct* metadata =
           envoy_config_core_v3_Node_mutable_metadata(node_msg, arena);
-      PopulateMetadata(metadata, node_->metadata(), arena);
+      PopulateMetadata(metadata, node->metadata(), arena);
     }
-    if (!node_->locality_region().empty() || !node_->locality_zone().empty() ||
-        !node_->locality_sub_zone().empty()) {
+    if (!node->locality_region().empty() || !node->locality_zone().empty() ||
+        !node->locality_sub_zone().empty()) {
       envoy_config_core_v3_Locality* locality =
           envoy_config_core_v3_Node_mutable_locality(node_msg, arena);
-      if (!node_->locality_region().empty()) {
+      if (!node->locality_region().empty()) {
         envoy_config_core_v3_Locality_set_region(
-            locality, StdStringToUpbString(node_->locality_region()));
+            locality, StdStringToUpbString(node->locality_region()));
       }
-      if (!node_->locality_zone().empty()) {
+      if (!node->locality_zone().empty()) {
         envoy_config_core_v3_Locality_set_zone(
-            locality, StdStringToUpbString(node_->locality_zone()));
+            locality, StdStringToUpbString(node->locality_zone()));
       }
-      if (!node_->locality_sub_zone().empty()) {
+      if (!node->locality_sub_zone().empty()) {
         envoy_config_core_v3_Locality_set_sub_zone(
-            locality, StdStringToUpbString(node_->locality_sub_zone()));
+            locality, StdStringToUpbString(node->locality_sub_zone()));
       }
     }
   }
   envoy_config_core_v3_Node_set_user_agent_name(
-      node_msg, StdStringToUpbString(user_agent_name_));
+      node_msg, StdStringToUpbString(user_agent_name));
   envoy_config_core_v3_Node_set_user_agent_version(
-      node_msg, StdStringToUpbString(user_agent_version_));
+      node_msg, StdStringToUpbString(user_agent_version));
   envoy_config_core_v3_Node_add_client_features(
       node_msg,
       upb_StringView_FromString("envoy.lb.does_not_support_overprovisioning"),
@@ -344,224 +350,4 @@ absl::Status XdsApi::ParseAdsResponse(absl::string_view encoded_response,
   return absl::OkStatus();
 }
 
-namespace {
-
-void MaybeLogLrsRequest(
-    const XdsApiContext& context,
-    const envoy_service_load_stats_v3_LoadStatsRequest* request) {
-  if (GRPC_TRACE_FLAG_ENABLED_OBJ(*context.tracer) && ABSL_VLOG_IS_ON(2)) {
-    const upb_MessageDef* msg_type =
-        envoy_service_load_stats_v3_LoadStatsRequest_getmsgdef(
-            context.def_pool);
-    char buf[10240];
-    upb_TextEncode(reinterpret_cast<const upb_Message*>(request), msg_type,
-                   nullptr, 0, buf, sizeof(buf));
-    VLOG(2) << "[xds_client " << context.client
-            << "] constructed LRS request: " << buf;
-  }
-}
-
-std::string SerializeLrsRequest(
-    const XdsApiContext& context,
-    const envoy_service_load_stats_v3_LoadStatsRequest* request) {
-  size_t output_length;
-  char* output = envoy_service_load_stats_v3_LoadStatsRequest_serialize(
-      request, context.arena, &output_length);
-  return std::string(output, output_length);
-}
-
-}  // namespace
-
-std::string XdsApi::CreateLrsInitialRequest() {
-  upb::Arena arena;
-  const XdsApiContext context = {client_, tracer_, def_pool_->ptr(),
-                                 arena.ptr()};
-  // Create a request.
-  envoy_service_load_stats_v3_LoadStatsRequest* request =
-      envoy_service_load_stats_v3_LoadStatsRequest_new(arena.ptr());
-  // Populate node.
-  envoy_config_core_v3_Node* node_msg =
-      envoy_service_load_stats_v3_LoadStatsRequest_mutable_node(request,
-                                                                arena.ptr());
-  PopulateNode(node_msg, arena.ptr());
-  envoy_config_core_v3_Node_add_client_features(
-      node_msg,
-      upb_StringView_FromString("envoy.lrs.supports_send_all_clusters"),
-      arena.ptr());
-  MaybeLogLrsRequest(context, request);
-  return SerializeLrsRequest(context, request);
-}
-
-namespace {
-
-void LocalityStatsPopulate(
-    const XdsApiContext& context,
-    envoy_config_endpoint_v3_UpstreamLocalityStats* output,
-    const XdsLocalityName& locality_name,
-    const XdsClusterLocalityStats::Snapshot& snapshot) {
-  // Set locality.
-  envoy_config_core_v3_Locality* locality =
-      envoy_config_endpoint_v3_UpstreamLocalityStats_mutable_locality(
-          output, context.arena);
-  if (!locality_name.region().empty()) {
-    envoy_config_core_v3_Locality_set_region(
-        locality, StdStringToUpbString(locality_name.region()));
-  }
-  if (!locality_name.zone().empty()) {
-    envoy_config_core_v3_Locality_set_zone(
-        locality, StdStringToUpbString(locality_name.zone()));
-  }
-  if (!locality_name.sub_zone().empty()) {
-    envoy_config_core_v3_Locality_set_sub_zone(
-        locality, StdStringToUpbString(locality_name.sub_zone()));
-  }
-  // Set total counts.
-  envoy_config_endpoint_v3_UpstreamLocalityStats_set_total_successful_requests(
-      output, snapshot.total_successful_requests);
-  envoy_config_endpoint_v3_UpstreamLocalityStats_set_total_requests_in_progress(
-      output, snapshot.total_requests_in_progress);
-  envoy_config_endpoint_v3_UpstreamLocalityStats_set_total_error_requests(
-      output, snapshot.total_error_requests);
-  envoy_config_endpoint_v3_UpstreamLocalityStats_set_total_issued_requests(
-      output, snapshot.total_issued_requests);
-  // Add backend metrics.
-  for (const auto& p : snapshot.backend_metrics) {
-    const std::string& metric_name = p.first;
-    const XdsClusterLocalityStats::BackendMetric& metric_value = p.second;
-    envoy_config_endpoint_v3_EndpointLoadMetricStats* load_metric =
-        envoy_config_endpoint_v3_UpstreamLocalityStats_add_load_metric_stats(
-            output, context.arena);
-    envoy_config_endpoint_v3_EndpointLoadMetricStats_set_metric_name(
-        load_metric, StdStringToUpbString(metric_name));
-    envoy_config_endpoint_v3_EndpointLoadMetricStats_set_num_requests_finished_with_metric(
-        load_metric, metric_value.num_requests_finished_with_metric);
-    envoy_config_endpoint_v3_EndpointLoadMetricStats_set_total_metric_value(
-        load_metric, metric_value.total_metric_value);
-  }
-}
-
-}  // namespace
-
-std::string XdsApi::CreateLrsRequest(
-    ClusterLoadReportMap cluster_load_report_map) {
-  upb::Arena arena;
-  const XdsApiContext context = {client_, tracer_, def_pool_->ptr(),
-                                 arena.ptr()};
-  // Create a request.
-  envoy_service_load_stats_v3_LoadStatsRequest* request =
-      envoy_service_load_stats_v3_LoadStatsRequest_new(arena.ptr());
-  for (auto& p : cluster_load_report_map) {
-    const std::string& cluster_name = p.first.first;
-    const std::string& eds_service_name = p.first.second;
-    const ClusterLoadReport& load_report = p.second;
-    // Add cluster stats.
-    envoy_config_endpoint_v3_ClusterStats* cluster_stats =
-        envoy_service_load_stats_v3_LoadStatsRequest_add_cluster_stats(
-            request, arena.ptr());
-    // Set the cluster name.
-    envoy_config_endpoint_v3_ClusterStats_set_cluster_name(
-        cluster_stats, StdStringToUpbString(cluster_name));
-    // Set EDS service name, if non-empty.
-    if (!eds_service_name.empty()) {
-      envoy_config_endpoint_v3_ClusterStats_set_cluster_service_name(
-          cluster_stats, StdStringToUpbString(eds_service_name));
-    }
-    // Add locality stats.
-    for (const auto& p : load_report.locality_stats) {
-      const XdsLocalityName& locality_name = *p.first;
-      const auto& snapshot = p.second;
-      envoy_config_endpoint_v3_UpstreamLocalityStats* locality_stats =
-          envoy_config_endpoint_v3_ClusterStats_add_upstream_locality_stats(
-              cluster_stats, arena.ptr());
-      LocalityStatsPopulate(context, locality_stats, locality_name, snapshot);
-    }
-    // Add dropped requests.
-    uint64_t total_dropped_requests = 0;
-    for (const auto& p : load_report.dropped_requests.categorized_drops) {
-      const std::string& category = p.first;
-      const uint64_t count = p.second;
-      envoy_config_endpoint_v3_ClusterStats_DroppedRequests* dropped_requests =
-          envoy_config_endpoint_v3_ClusterStats_add_dropped_requests(
-              cluster_stats, arena.ptr());
-      envoy_config_endpoint_v3_ClusterStats_DroppedRequests_set_category(
-          dropped_requests, StdStringToUpbString(category));
-      envoy_config_endpoint_v3_ClusterStats_DroppedRequests_set_dropped_count(
-          dropped_requests, count);
-      total_dropped_requests += count;
-    }
-    total_dropped_requests += load_report.dropped_requests.uncategorized_drops;
-    // Set total dropped requests.
-    envoy_config_endpoint_v3_ClusterStats_set_total_dropped_requests(
-        cluster_stats, total_dropped_requests);
-    // Set real load report interval.
-    gpr_timespec timespec = load_report.load_report_interval.as_timespec();
-    google_protobuf_Duration* load_report_interval =
-        envoy_config_endpoint_v3_ClusterStats_mutable_load_report_interval(
-            cluster_stats, arena.ptr());
-    google_protobuf_Duration_set_seconds(load_report_interval, timespec.tv_sec);
-    google_protobuf_Duration_set_nanos(load_report_interval, timespec.tv_nsec);
-  }
-  MaybeLogLrsRequest(context, request);
-  return SerializeLrsRequest(context, request);
-}
-
-namespace {
-
-void MaybeLogLrsResponse(
-    const XdsApiContext& context,
-    const envoy_service_load_stats_v3_LoadStatsResponse* response) {
-  if (GRPC_TRACE_FLAG_ENABLED_OBJ(*context.tracer) && ABSL_VLOG_IS_ON(2)) {
-    const upb_MessageDef* msg_type =
-        envoy_service_load_stats_v3_LoadStatsResponse_getmsgdef(
-            context.def_pool);
-    char buf[10240];
-    upb_TextEncode(reinterpret_cast<const upb_Message*>(response), msg_type,
-                   nullptr, 0, buf, sizeof(buf));
-    VLOG(2) << "[xds_client " << context.client
-            << "] received LRS response: " << buf;
-  }
-}
-
-}  // namespace
-
-absl::Status XdsApi::ParseLrsResponse(absl::string_view encoded_response,
-                                      bool* send_all_clusters,
-                                      std::set<std::string>* cluster_names,
-                                      Duration* load_reporting_interval) {
-  upb::Arena arena;
-  // Decode the response.
-  const envoy_service_load_stats_v3_LoadStatsResponse* decoded_response =
-      envoy_service_load_stats_v3_LoadStatsResponse_parse(
-          encoded_response.data(), encoded_response.size(), arena.ptr());
-  // Parse the response.
-  if (decoded_response == nullptr) {
-    return absl::UnavailableError("Can't decode response.");
-  }
-  const XdsApiContext context = {client_, tracer_, def_pool_->ptr(),
-                                 arena.ptr()};
-  MaybeLogLrsResponse(context, decoded_response);
-  // Check send_all_clusters.
-  if (envoy_service_load_stats_v3_LoadStatsResponse_send_all_clusters(
-          decoded_response)) {
-    *send_all_clusters = true;
-  } else {
-    // Store the cluster names.
-    size_t size;
-    const upb_StringView* clusters =
-        envoy_service_load_stats_v3_LoadStatsResponse_clusters(decoded_response,
-                                                               &size);
-    for (size_t i = 0; i < size; ++i) {
-      cluster_names->emplace(UpbStringToStdString(clusters[i]));
-    }
-  }
-  // Get the load report interval.
-  const google_protobuf_Duration* load_reporting_interval_duration =
-      envoy_service_load_stats_v3_LoadStatsResponse_load_reporting_interval(
-          decoded_response);
-  *load_reporting_interval = Duration::FromSecondsAndNanoseconds(
-      google_protobuf_Duration_seconds(load_reporting_interval_duration),
-      google_protobuf_Duration_nanos(load_reporting_interval_duration));
-  return absl::OkStatus();
-}
-
 }  // namespace grpc_core

data/src/core/xds/xds_client/xds_api.h

@@ -17,6 +17,7 @@
 #ifndef GRPC_SRC_CORE_XDS_XDS_CLIENT_XDS_API_H
 #define GRPC_SRC_CORE_XDS_XDS_CLIENT_XDS_API_H
 
+#include <grpc/support/port_platform.h>
 #include <stddef.h>
 
 #include <map>
@@ -29,25 +30,20 @@
 #include "absl/strings/string_view.h"
 #include "envoy/admin/v3/config_dump_shared.upb.h"
 #include "envoy/service/status/v3/csds.upb.h"
-#include "upb/mem/arena.h"
-#include "upb/reflection/def.hpp"
-
-#include <grpc/support/port_platform.h>
-
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gprpp/ref_counted_ptr.h"
-#include "src/core/lib/gprpp/time.h"
+#include "src/core/util/ref_counted_ptr.h"
+#include "src/core/util/time.h"
 #include "src/core/xds/xds_client/xds_bootstrap.h"
-#include "src/core/xds/xds_client/xds_client_stats.h"
+#include "src/core/xds/xds_client/xds_locality.h"
+#include "upb/mem/arena.h"
+#include "upb/reflection/def.hpp"
 
 namespace grpc_core {
 
 class XdsClient;
 
-// TODO(roth): When we have time, split this into multiple pieces:
-// - ADS request/response handling
-// - LRS request/response handling
-// - CSDS response generation
+// TODO(roth): When we have time, remove this class and move its
+// functionality directly inside of XdsClient.
 class XdsApi final {
  public:
   // Interface defined by caller and passed to ParseAdsResponse().
@@ -81,17 +77,6 @@ class XdsApi final {
                                               absl::string_view message) = 0;
   };
 
-  struct ClusterLoadReport {
-    XdsClusterDropStats::Snapshot dropped_requests;
-    std::map<RefCountedPtr<XdsLocalityName>, XdsClusterLocalityStats::Snapshot,
-             XdsLocalityName::Less>
-        locality_stats;
-    Duration load_report_interval;
-  };
-  using ClusterLoadReportMap = std::map<
-      std::pair<std::string /*cluster_name*/, std::string /*eds_service_name*/>,
-      ClusterLoadReport>;
-
   // The metadata of the xDS resource; used by the xDS config dump.
   struct ResourceMetadata {
     // Resource status from the view of a xDS client, which tells the
@@ -160,19 +145,6 @@ class XdsApi final {
   absl::Status ParseAdsResponse(absl::string_view encoded_response,
                                 AdsResponseParserInterface* parser);
 
-  // Creates an initial LRS request.
-  std::string CreateLrsInitialRequest();
-
-  // Creates an LRS request sending a client-side load report.
-  std::string CreateLrsRequest(ClusterLoadReportMap cluster_load_report_map);
-
-  // Parses the LRS response and populates send_all_clusters,
-  // cluster_names, and load_reporting_interval.
-  absl::Status ParseLrsResponse(absl::string_view encoded_response,
-                                bool* send_all_clusters,
-                                std::set<std::string>* cluster_names,
-                                Duration* load_reporting_interval);
-
   void PopulateNode(envoy_config_core_v3_Node* node_msg, upb_Arena* arena);
 
  private:
@@ -184,6 +156,11 @@ class XdsApi final {
   const std::string user_agent_version_;
 };
 
+void PopulateXdsNode(const XdsBootstrap::Node* node,
+                     absl::string_view user_agent_name,
+                     absl::string_view user_agent_version,
+                     envoy_config_core_v3_Node* node_msg, upb_Arena* arena);
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_XDS_XDS_CLIENT_XDS_API_H

data/src/core/xds/xds_client/xds_backend_metric_propagation.cc

@@ -0,0 +1,63 @@
+//
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "src/core/xds/xds_client/xds_backend_metric_propagation.h"
+
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
+#include "src/core/util/useful.h"
+
+namespace grpc_core {
+
+std::string BackendMetricPropagation::AsString() const {
+  std::vector<std::string> parts;
+  if (propagation_bits & kCpuUtilization) parts.push_back("cpu_utilization");
+  if (propagation_bits & kMemUtilization) parts.push_back("mem_utilization");
+  if (propagation_bits & kApplicationUtilization) {
+    parts.push_back("application_utilization");
+  }
+  if (propagation_bits & kNamedMetricsAll) {
+    parts.push_back("named_metrics.*");
+  } else {
+    // Output keys in sorted order for consistency.
+    std::vector<absl::string_view> keys(named_metric_keys.begin(),
+                                        named_metric_keys.end());
+    std::sort(keys.begin(), keys.end());
+    for (const auto& key : keys) {
+      parts.push_back(absl::StrCat("named_metrics.", key));
+    }
+  }
+  return absl::StrCat("{", absl::StrJoin(parts, ","), "}");
+}
+
+bool BackendMetricPropagation::operator<(
+    const BackendMetricPropagation& other) const {
+  int c = QsortCompare(propagation_bits, other.propagation_bits);
+  if (c != 0) return c == -1;
+  auto other_it = other.named_metric_keys.begin();
+  for (auto it = named_metric_keys.begin(); it != named_metric_keys.end();
+       ++it) {
+    if (other_it == other.named_metric_keys.end()) return false;
+    c = QsortCompare(*it, *other_it);
+    if (c != 0) return c == -1;
+    ++other_it;
+  }
+  return false;
+}
+
+}  // namespace grpc_core

data/src/core/xds/xds_client/xds_backend_metric_propagation.h

@@ -0,0 +1,59 @@
+//
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef GRPC_SRC_CORE_XDS_XDS_CLIENT_XDS_BACKEND_METRIC_PROPAGATION_H
+#define GRPC_SRC_CORE_XDS_XDS_CLIENT_XDS_BACKEND_METRIC_PROPAGATION_H
+
+#include <string>
+
+#include "absl/container/flat_hash_set.h"
+#include "src/core/util/ref_counted.h"
+#include "src/core/util/ref_counted_ptr.h"
+
+namespace grpc_core {
+
+struct BackendMetricPropagation : public RefCounted<BackendMetricPropagation> {
+  static constexpr uint8_t kCpuUtilization = 1;
+  static constexpr uint8_t kMemUtilization = 2;
+  static constexpr uint8_t kApplicationUtilization = 4;
+  static constexpr uint8_t kNamedMetricsAll = 8;
+
+  uint8_t propagation_bits = 0;
+  absl::flat_hash_set<std::string> named_metric_keys;
+
+  std::string AsString() const;
+
+  bool operator==(const BackendMetricPropagation& other) const {
+    return propagation_bits == other.propagation_bits &&
+           named_metric_keys == other.named_metric_keys;
+  }
+
+  bool operator<(const BackendMetricPropagation& other) const;
+
+  // Sorting functor for RefCountedPtr<const BackendMetricPropagation>.
+  struct Less {
+    bool operator()(
+        const RefCountedPtr<const BackendMetricPropagation>& p1,
+        const RefCountedPtr<const BackendMetricPropagation>& p2) const {
+      if (p1 == nullptr || p2 == nullptr) return p1.get() < p2.get();
+      return *p1 < *p2;
+    }
+  };
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_XDS_XDS_CLIENT_XDS_BACKEND_METRIC_PROPAGATION_H

data/src/core/xds/xds_client/xds_bootstrap.cc

@@ -16,11 +16,10 @@
 
 #include "src/core/xds/xds_client/xds_bootstrap.h"
 
-#include "absl/types/optional.h"
-
 #include <grpc/support/port_platform.h>
 
-#include "src/core/lib/gprpp/env.h"
+#include "absl/types/optional.h"
+#include "src/core/util/env.h"
 #include "src/core/util/string.h"
 
 namespace grpc_core {

data/src/core/xds/xds_client/xds_bootstrap.h

@@ -17,10 +17,10 @@
 #ifndef GRPC_SRC_CORE_XDS_XDS_CLIENT_XDS_BOOTSTRAP_H
 #define GRPC_SRC_CORE_XDS_XDS_CLIENT_XDS_BOOTSTRAP_H
 
-#include <string>
-
 #include <grpc/support/port_platform.h>
 
+#include <string>
+
 #include "src/core/util/json/json.h"
 
 namespace grpc_core {