grpc 1.54.3 → 1.59.2

data/src/core/lib/config/config_vars.h

@@ -31,22 +31,22 @@
 
 namespace grpc_core {
 
-class ConfigVars {
+class GPR_DLL ConfigVars {
  public:
   struct Overrides {
     absl::optional<int32_t> client_channel_backup_poll_interval_ms;
     absl::optional<bool> enable_fork_support;
     absl::optional<bool> abort_on_leaks;
     absl::optional<bool> not_use_system_ssl_roots;
-    absl::optional<std::string> experiments;
     absl::optional<std::string> dns_resolver;
-    absl::optional<std::string> trace;
     absl::optional<std::string> verbosity;
     absl::optional<std::string> stacktrace_minloglevel;
     absl::optional<std::string> poll_strategy;
     absl::optional<std::string> system_ssl_roots_dir;
     absl::optional<std::string> default_ssl_roots_file_path;
     absl::optional<std::string> ssl_cipher_suites;
+    absl::optional<std::string> experiments;
+    absl::optional<std::string> trace;
   };
   ConfigVars(const ConfigVars&) = delete;
   ConfigVars& operator=(const ConfigVars&) = delete;
@@ -111,13 +111,13 @@ class ConfigVars {
   bool enable_fork_support_;
   bool abort_on_leaks_;
   bool not_use_system_ssl_roots_;
-  std::string experiments_;
   std::string dns_resolver_;
-  std::string trace_;
   std::string verbosity_;
   std::string stacktrace_minloglevel_;
   std::string poll_strategy_;
   std::string ssl_cipher_suites_;
+  std::string experiments_;
+  std::string trace_;
   absl::optional<std::string> override_system_ssl_roots_dir_;
   absl::optional<std::string> override_default_ssl_roots_file_path_;
 };

data/src/core/lib/config/core_configuration.cc

@@ -49,7 +49,8 @@ CoreConfiguration::CoreConfiguration(Builder* builder)
       certificate_provider_registry_(
           builder->certificate_provider_registry_.Build()) {}
 
-void CoreConfiguration::RegisterBuilder(std::function<void(Builder*)> builder) {
+void CoreConfiguration::RegisterBuilder(
+    absl::AnyInvocable<void(Builder*)> builder) {
   GPR_ASSERT(config_.load(std::memory_order_relaxed) == nullptr &&
              "CoreConfiguration was already instantiated before builder "
              "registration was completed");

data/src/core/lib/config/core_configuration.h

@@ -18,7 +18,8 @@
 #include <grpc/support/port_platform.h>
 
 #include <atomic>
-#include <functional>
+
+#include "absl/functional/any_invocable.h"
 
 #include <grpc/support/log.h>
 
@@ -36,7 +37,7 @@ namespace grpc_core {
 
 // Global singleton that stores library configuration - factories, etc...
 // that plugins might choose to extend.
-class CoreConfiguration {
+class GRPC_DLL CoreConfiguration {
  public:
   CoreConfiguration(const CoreConfiguration&) = delete;
   CoreConfiguration& operator=(const CoreConfiguration&) = delete;
@@ -98,7 +99,7 @@ class CoreConfiguration {
 
   // Stores a builder for RegisterBuilder
   struct RegisteredBuilder {
-    std::function<void(Builder*)> builder;
+    absl::AnyInvocable<void(Builder*)> builder;
     RegisteredBuilder* next;
   };
 
@@ -154,7 +155,7 @@ class CoreConfiguration {
   // Attach a registration function globally.
   // Each registration function is called *in addition to*
   // BuildCoreConfiguration for the default core configuration.
-  static void RegisterBuilder(std::function<void(Builder*)> builder);
+  static void RegisterBuilder(absl::AnyInvocable<void(Builder*)> builder);
 
   // Drop the core configuration. Users must ensure no other threads are
   // accessing the configuration.

data/src/core/lib/config/load_config.cc

@@ -20,8 +20,10 @@
 
 #include "absl/flags/marshalling.h"
 #include "absl/strings/numbers.h"
+#include "absl/strings/str_join.h"
 #include "absl/types/optional.h"
 
+#include "src/core/lib/gpr/log_internal.h"
 #include "src/core/lib/gprpp/env.h"
 
 namespace grpc_core {
@@ -34,6 +36,7 @@ absl::optional<std::string> LoadEnv(absl::string_view environment_variable) {
 
 std::string LoadConfigFromEnv(absl::string_view environment_variable,
                               const char* default_value) {
+  GPR_ASSERT_INTERNAL(!environment_variable.empty());
   return LoadEnv(environment_variable).value_or(default_value);
 }
 
@@ -63,4 +66,14 @@ bool LoadConfigFromEnv(absl::string_view environment_variable,
   return default_value;
 }
 
+std::string LoadConfig(const absl::Flag<std::vector<std::string>>& flag,
+                       absl::string_view environment_variable,
+                       const absl::optional<std::string>& override,
+                       const char* default_value) {
+  if (override.has_value()) return *override;
+  auto from_flag = absl::GetFlag(flag);
+  if (!from_flag.empty()) return absl::StrJoin(from_flag, ",");
+  return LoadConfigFromEnv(environment_variable, default_value);
+}
+
 }  // namespace grpc_core

data/src/core/lib/config/load_config.h

@@ -20,6 +20,7 @@
 #include <stdint.h>
 
 #include <string>
+#include <vector>
 
 #include "absl/flags/flag.h"
 #include "absl/strings/string_view.h"
@@ -44,6 +45,11 @@ T LoadConfig(const absl::Flag<absl::optional<T>>& flag,
   return LoadConfigFromEnv(environment_variable, default_value);
 }
 
+std::string LoadConfig(const absl::Flag<std::vector<std::string>>& flag,
+                       absl::string_view environment_variable,
+                       const absl::optional<std::string>& override,
+                       const char* default_value);
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_CONFIG_LOAD_CONFIG_H

data/src/core/lib/debug/event_log.h

@@ -71,7 +71,7 @@ class EventLog {
   std::vector<Entry> EndCollection(
       absl::Span<const absl::string_view> wanted_events);
 
-  PerCpu<Fragment> fragments_;
+  PerCpu<Fragment> fragments_{PerCpuOptions().SetCpusPerShard(2)};
   gpr_cycle_counter collection_begin_;
   static std::atomic<EventLog*> g_instance_;
 };

data/src/core/lib/debug/stats_data.cc

@@ -27,6 +27,19 @@ union DblUint {
   uint64_t uint;
 };
 }  // namespace
+void HistogramCollector_100000_20::Collect(Histogram_100000_20* result) const {
+  for (int i = 0; i < 20; i++) {
+    result->buckets_[i] += buckets_[i].load(std::memory_order_relaxed);
+  }
+}
+Histogram_100000_20 operator-(const Histogram_100000_20& left,
+                              const Histogram_100000_20& right) {
+  Histogram_100000_20 result;
+  for (int i = 0; i < 20; i++) {
+    result.buckets_[i] = left.buckets_[i] - right.buckets_[i];
+  }
+  return result;
+}
 void HistogramCollector_65536_26::Collect(Histogram_65536_26* result) const {
   for (int i = 0; i < 26; i++) {
     result->buckets_[i] += buckets_[i].load(std::memory_order_relaxed);
@@ -67,17 +80,42 @@ Histogram_80_10 operator-(const Histogram_80_10& left,
   }
   return result;
 }
+void HistogramCollector_10000_20::Collect(Histogram_10000_20* result) const {
+  for (int i = 0; i < 20; i++) {
+    result->buckets_[i] += buckets_[i].load(std::memory_order_relaxed);
+  }
+}
+Histogram_10000_20 operator-(const Histogram_10000_20& left,
+                             const Histogram_10000_20& right) {
+  Histogram_10000_20 result;
+  for (int i = 0; i < 20; i++) {
+    result.buckets_[i] = left.buckets_[i] - right.buckets_[i];
+  }
+  return result;
+}
 const absl::string_view
     GlobalStats::counter_name[static_cast<int>(Counter::COUNT)] = {
-        "client_calls_created",    "server_calls_created",
-        "client_channels_created", "client_subchannels_created",
-        "server_channels_created", "insecure_connections_created",
-        "syscall_write",           "syscall_read",
-        "tcp_read_alloc_8k",       "tcp_read_alloc_64k",
-        "http2_settings_writes",   "http2_pings_sent",
-        "http2_writes_begun",      "http2_transport_stalls",
-        "http2_stream_stalls",     "cq_pluck_creates",
-        "cq_next_creates",         "cq_callback_creates",
+        "client_calls_created",
+        "server_calls_created",
+        "client_channels_created",
+        "client_subchannels_created",
+        "server_channels_created",
+        "insecure_connections_created",
+        "syscall_write",
+        "syscall_read",
+        "tcp_read_alloc_8k",
+        "tcp_read_alloc_64k",
+        "http2_settings_writes",
+        "http2_pings_sent",
+        "http2_writes_begun",
+        "http2_transport_stalls",
+        "http2_stream_stalls",
+        "cq_pluck_creates",
+        "cq_next_creates",
+        "cq_callback_creates",
+        "wrr_updates",
+        "work_serializer_items_enqueued",
+        "work_serializer_items_dequeued",
 };
 const absl::string_view GlobalStats::counter_doc[static_cast<int>(
     Counter::COUNT)] = {
@@ -105,12 +143,26 @@ const absl::string_view GlobalStats::counter_doc[static_cast<int>(
     "usage)",
     "Number of completion queues created for cq_callback (indicates callback "
     "api usage)",
+    "Number of wrr updates that have been received",
+    "Number of items enqueued onto work serializers",
+    "Number of items dequeued from work serializers",
 };
-const absl::string_view GlobalStats::histogram_name[static_cast<int>(
-    Histogram::COUNT)] = {
-    "call_initial_size",       "tcp_write_size",      "tcp_write_iov_size",
-    "tcp_read_size",           "tcp_read_offer",      "tcp_read_offer_iov_size",
-    "http2_send_message_size", "http2_metadata_size",
+const absl::string_view
+    GlobalStats::histogram_name[static_cast<int>(Histogram::COUNT)] = {
+        "call_initial_size",
+        "tcp_write_size",
+        "tcp_write_iov_size",
+        "tcp_read_size",
+        "tcp_read_offer",
+        "tcp_read_offer_iov_size",
+        "http2_send_message_size",
+        "http2_metadata_size",
+        "wrr_subchannel_list_size",
+        "wrr_subchannel_ready_size",
+        "work_serializer_run_time_ms",
+        "work_serializer_work_time_ms",
+        "work_serializer_work_time_per_item_ms",
+        "work_serializer_items_per_run",
 };
 const absl::string_view GlobalStats::histogram_doc[static_cast<int>(
     Histogram::COUNT)] = {
@@ -122,26 +174,45 @@ const absl::string_view GlobalStats::histogram_doc[static_cast<int>(
     "Number of byte segments offered to each syscall_read",
     "Size of messages received by HTTP2 transport",
     "Number of bytes consumed by metadata, according to HPACK accounting rules",
+    "Number of subchannels in a subchannel list at picker creation time",
+    "Number of READY subchannels in a subchannel list at picker creation time",
+    "Number of milliseconds work serializers run for",
+    "When running, how many milliseconds are work serializers actually doing "
+    "work",
+    "How long do individual items take to process in work serializers",
+    "How many callbacks are executed when a work serializer runs",
 };
 namespace {
-const int kStatsTable0[27] = {0,    1,     2,     4,     7,     11,   17,
+const int kStatsTable0[21] = {0,    1,    2,    4,     8,     15,    27,
+                              49,   89,   160,  288,   517,   928,   1666,
+                              2991, 5369, 9637, 17297, 31045, 55719, 100000};
+const uint8_t kStatsTable1[30] = {3,  3,  4,  4,  5,  6,  6,  7,  7,  8,
+                                  9,  9,  10, 10, 11, 11, 12, 13, 13, 14,
+                                  15, 15, 16, 16, 17, 17, 18, 19, 19, 20};
+const int kStatsTable2[27] = {0,    1,     2,     4,     7,     11,   17,
                               26,   40,    61,    92,    139,   210,  317,
                               478,  721,   1087,  1638,  2468,  3719, 5604,
                               8443, 12721, 19166, 28875, 43502, 65536};
-const uint8_t kStatsTable1[29] = {3,  3,  4,  5,  6,  6,  7,  8,  9,  10,
+const uint8_t kStatsTable3[29] = {3,  3,  4,  5,  6,  6,  7,  8,  9,  10,
                                   11, 11, 12, 13, 14, 15, 16, 16, 17, 18,
                                   19, 20, 21, 21, 22, 23, 24, 25, 26};
-const int kStatsTable2[21] = {
+const int kStatsTable4[21] = {
     0,     1,      3,      8,       19,      45,      106,
     250,   588,    1383,   3252,    7646,    17976,   42262,
     99359, 233593, 549177, 1291113, 3035402, 7136218, 16777216};
-const uint8_t kStatsTable3[23] = {2,  3,  3,  4,  5,  6,  7,  8,
+const uint8_t kStatsTable5[23] = {2,  3,  3,  4,  5,  6,  7,  8,
                                   8,  9,  10, 11, 12, 12, 13, 14,
                                   15, 16, 16, 17, 18, 19, 20};
-const int kStatsTable4[11] = {0, 1, 2, 4, 7, 11, 17, 26, 38, 56, 80};
-const uint8_t kStatsTable5[9] = {3, 3, 4, 5, 6, 6, 7, 8, 9};
+const int kStatsTable6[11] = {0, 1, 2, 4, 7, 11, 17, 26, 38, 56, 80};
+const uint8_t kStatsTable7[9] = {3, 3, 4, 5, 6, 6, 7, 8, 9};
+const int kStatsTable8[21] = {0,   1,    2,    4,    7,    12,   19,
+                              30,  47,   74,   116,  182,  285,  445,
+                              695, 1084, 1691, 2637, 4113, 6414, 10000};
+const uint8_t kStatsTable9[23] = {3,  3,  4,  5,  5,  6,  7,  8,
+                                  9,  9,  10, 11, 12, 12, 13, 14,
+                                  15, 15, 16, 17, 18, 18, 19};
 }  // namespace
-int Histogram_65536_26::BucketFor(int value) {
+int Histogram_100000_20::BucketFor(int value) {
   if (value < 3) {
     if (value < 0) {
       return 0;
@@ -149,12 +220,31 @@ int Histogram_65536_26::BucketFor(int value) {
       return value;
     }
   } else {
-    if (value < 49153) {
+    if (value < 65537) {
       DblUint val;
       val.dbl = value;
       const int bucket =
           kStatsTable1[((val.uint - 4613937818241073152ull) >> 51)];
       return bucket - (value < kStatsTable0[bucket]);
+    } else {
+      return 19;
+    }
+  }
+}
+int Histogram_65536_26::BucketFor(int value) {
+  if (value < 3) {
+    if (value < 0) {
+      return 0;
+    } else {
+      return value;
+    }
+  } else {
+    if (value < 49153) {
+      DblUint val;
+      val.dbl = value;
+      const int bucket =
+          kStatsTable3[((val.uint - 4613937818241073152ull) >> 51)];
+      return bucket - (value < kStatsTable2[bucket]);
     } else {
       return 25;
     }
@@ -172,8 +262,8 @@ int Histogram_16777216_20::BucketFor(int value) {
       DblUint val;
       val.dbl = value;
       const int bucket =
-          kStatsTable3[((val.uint - 4611686018427387904ull) >> 52)];
-      return bucket - (value < kStatsTable2[bucket]);
+          kStatsTable5[((val.uint - 4611686018427387904ull) >> 52)];
+      return bucket - (value < kStatsTable4[bucket]);
     } else {
       return 19;
     }
@@ -191,8 +281,8 @@ int Histogram_80_10::BucketFor(int value) {
       DblUint val;
       val.dbl = value;
       const int bucket =
-          kStatsTable5[((val.uint - 4613937818241073152ull) >> 51)];
-      return bucket - (value < kStatsTable4[bucket]);
+          kStatsTable7[((val.uint - 4613937818241073152ull) >> 51)];
+      return bucket - (value < kStatsTable6[bucket]);
     } else {
       if (value < 56) {
         return 8;
@@ -202,6 +292,29 @@ int Histogram_80_10::BucketFor(int value) {
     }
   }
 }
+int Histogram_10000_20::BucketFor(int value) {
+  if (value < 3) {
+    if (value < 0) {
+      return 0;
+    } else {
+      return value;
+    }
+  } else {
+    if (value < 6145) {
+      DblUint val;
+      val.dbl = value;
+      const int bucket =
+          kStatsTable9[((val.uint - 4613937818241073152ull) >> 51)];
+      return bucket - (value < kStatsTable8[bucket]);
+    } else {
+      if (value < 6414) {
+        return 18;
+      } else {
+        return 19;
+      }
+    }
+  }
+}
 GlobalStats::GlobalStats()
     : client_calls_created{0},
       server_calls_created{0},
@@ -220,35 +333,56 @@ GlobalStats::GlobalStats()
       http2_stream_stalls{0},
       cq_pluck_creates{0},
       cq_next_creates{0},
-      cq_callback_creates{0} {}
+      cq_callback_creates{0},
+      wrr_updates{0},
+      work_serializer_items_enqueued{0},
+      work_serializer_items_dequeued{0} {}
 HistogramView GlobalStats::histogram(Histogram which) const {
   switch (which) {
     default:
       GPR_UNREACHABLE_CODE(return HistogramView());
     case Histogram::kCallInitialSize:
-      return HistogramView{&Histogram_65536_26::BucketFor, kStatsTable0, 26,
+      return HistogramView{&Histogram_65536_26::BucketFor, kStatsTable2, 26,
                            call_initial_size.buckets()};
     case Histogram::kTcpWriteSize:
-      return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable2, 20,
+      return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable4, 20,
                            tcp_write_size.buckets()};
     case Histogram::kTcpWriteIovSize:
-      return HistogramView{&Histogram_80_10::BucketFor, kStatsTable4, 10,
+      return HistogramView{&Histogram_80_10::BucketFor, kStatsTable6, 10,
                            tcp_write_iov_size.buckets()};
     case Histogram::kTcpReadSize:
-      return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable2, 20,
+      return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable4, 20,
                            tcp_read_size.buckets()};
     case Histogram::kTcpReadOffer:
-      return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable2, 20,
+      return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable4, 20,
                            tcp_read_offer.buckets()};
     case Histogram::kTcpReadOfferIovSize:
-      return HistogramView{&Histogram_80_10::BucketFor, kStatsTable4, 10,
+      return HistogramView{&Histogram_80_10::BucketFor, kStatsTable6, 10,
                            tcp_read_offer_iov_size.buckets()};
     case Histogram::kHttp2SendMessageSize:
-      return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable2, 20,
+      return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable4, 20,
                            http2_send_message_size.buckets()};
     case Histogram::kHttp2MetadataSize:
-      return HistogramView{&Histogram_65536_26::BucketFor, kStatsTable0, 26,
+      return HistogramView{&Histogram_65536_26::BucketFor, kStatsTable2, 26,
                            http2_metadata_size.buckets()};
+    case Histogram::kWrrSubchannelListSize:
+      return HistogramView{&Histogram_10000_20::BucketFor, kStatsTable8, 20,
+                           wrr_subchannel_list_size.buckets()};
+    case Histogram::kWrrSubchannelReadySize:
+      return HistogramView{&Histogram_10000_20::BucketFor, kStatsTable8, 20,
+                           wrr_subchannel_ready_size.buckets()};
+    case Histogram::kWorkSerializerRunTimeMs:
+      return HistogramView{&Histogram_100000_20::BucketFor, kStatsTable0, 20,
+                           work_serializer_run_time_ms.buckets()};
+    case Histogram::kWorkSerializerWorkTimeMs:
+      return HistogramView{&Histogram_100000_20::BucketFor, kStatsTable0, 20,
+                           work_serializer_work_time_ms.buckets()};
+    case Histogram::kWorkSerializerWorkTimePerItemMs:
+      return HistogramView{&Histogram_100000_20::BucketFor, kStatsTable0, 20,
+                           work_serializer_work_time_per_item_ms.buckets()};
+    case Histogram::kWorkSerializerItemsPerRun:
+      return HistogramView{&Histogram_10000_20::BucketFor, kStatsTable8, 20,
+                           work_serializer_items_per_run.buckets()};
   }
 }
 std::unique_ptr<GlobalStats> GlobalStatsCollector::Collect() const {
@@ -288,6 +422,11 @@ std::unique_ptr<GlobalStats> GlobalStatsCollector::Collect() const {
         data.cq_next_creates.load(std::memory_order_relaxed);
     result->cq_callback_creates +=
         data.cq_callback_creates.load(std::memory_order_relaxed);
+    result->wrr_updates += data.wrr_updates.load(std::memory_order_relaxed);
+    result->work_serializer_items_enqueued +=
+        data.work_serializer_items_enqueued.load(std::memory_order_relaxed);
+    result->work_serializer_items_dequeued +=
+        data.work_serializer_items_dequeued.load(std::memory_order_relaxed);
     data.call_initial_size.Collect(&result->call_initial_size);
     data.tcp_write_size.Collect(&result->tcp_write_size);
     data.tcp_write_iov_size.Collect(&result->tcp_write_iov_size);
@@ -296,6 +435,16 @@ std::unique_ptr<GlobalStats> GlobalStatsCollector::Collect() const {
     data.tcp_read_offer_iov_size.Collect(&result->tcp_read_offer_iov_size);
     data.http2_send_message_size.Collect(&result->http2_send_message_size);
     data.http2_metadata_size.Collect(&result->http2_metadata_size);
+    data.wrr_subchannel_list_size.Collect(&result->wrr_subchannel_list_size);
+    data.wrr_subchannel_ready_size.Collect(&result->wrr_subchannel_ready_size);
+    data.work_serializer_run_time_ms.Collect(
+        &result->work_serializer_run_time_ms);
+    data.work_serializer_work_time_ms.Collect(
+        &result->work_serializer_work_time_ms);
+    data.work_serializer_work_time_per_item_ms.Collect(
+        &result->work_serializer_work_time_per_item_ms);
+    data.work_serializer_items_per_run.Collect(
+        &result->work_serializer_items_per_run);
   }
   return result;
 }
@@ -327,6 +476,11 @@ std::unique_ptr<GlobalStats> GlobalStats::Diff(const GlobalStats& other) const {
   result->cq_pluck_creates = cq_pluck_creates - other.cq_pluck_creates;
   result->cq_next_creates = cq_next_creates - other.cq_next_creates;
   result->cq_callback_creates = cq_callback_creates - other.cq_callback_creates;
+  result->wrr_updates = wrr_updates - other.wrr_updates;
+  result->work_serializer_items_enqueued =
+      work_serializer_items_enqueued - other.work_serializer_items_enqueued;
+  result->work_serializer_items_dequeued =
+      work_serializer_items_dequeued - other.work_serializer_items_dequeued;
   result->call_initial_size = call_initial_size - other.call_initial_size;
   result->tcp_write_size = tcp_write_size - other.tcp_write_size;
   result->tcp_write_iov_size = tcp_write_iov_size - other.tcp_write_iov_size;
@@ -337,6 +491,19 @@ std::unique_ptr<GlobalStats> GlobalStats::Diff(const GlobalStats& other) const {
   result->http2_send_message_size =
       http2_send_message_size - other.http2_send_message_size;
   result->http2_metadata_size = http2_metadata_size - other.http2_metadata_size;
+  result->wrr_subchannel_list_size =
+      wrr_subchannel_list_size - other.wrr_subchannel_list_size;
+  result->wrr_subchannel_ready_size =
+      wrr_subchannel_ready_size - other.wrr_subchannel_ready_size;
+  result->work_serializer_run_time_ms =
+      work_serializer_run_time_ms - other.work_serializer_run_time_ms;
+  result->work_serializer_work_time_ms =
+      work_serializer_work_time_ms - other.work_serializer_work_time_ms;
+  result->work_serializer_work_time_per_item_ms =
+      work_serializer_work_time_per_item_ms -
+      other.work_serializer_work_time_per_item_ms;
+  result->work_serializer_items_per_run =
+      work_serializer_items_per_run - other.work_serializer_items_per_run;
   return result;
 }
 }  // namespace grpc_core