grpc 1.48.0 → 1.50.0

data/src/core/lib/channel/promise_based_filter.cc

@@ -22,7 +22,6 @@
 #include "absl/base/attributes.h"
 #include "absl/memory/memory.h"
 #include "absl/types/variant.h"
-#include "absl/utility/utility.h"
 
 #include <grpc/status.h>
 
@@ -134,7 +133,7 @@ BaseCallData::CapturedBatch& BaseCallData::CapturedBatch::operator=(
 }
 
 void BaseCallData::CapturedBatch::ResumeWith(Flusher* releaser) {
-  auto* batch = absl::exchange(batch_, nullptr);
+  auto* batch = std::exchange(batch_, nullptr);
   GPR_ASSERT(batch != nullptr);
   uintptr_t& refcnt = *RefCountField(batch);
   if (refcnt == 0) return;  // refcnt==0 ==> cancelled
@@ -144,7 +143,7 @@ void BaseCallData::CapturedBatch::ResumeWith(Flusher* releaser) {
 }
 
 void BaseCallData::CapturedBatch::CompleteWith(Flusher* releaser) {
-  auto* batch = absl::exchange(batch_, nullptr);
+  auto* batch = std::exchange(batch_, nullptr);
   GPR_ASSERT(batch != nullptr);
   uintptr_t& refcnt = *RefCountField(batch);
   if (refcnt == 0) return;  // refcnt==0 ==> cancelled
@@ -155,7 +154,7 @@ void BaseCallData::CapturedBatch::CompleteWith(Flusher* releaser) {
 
 void BaseCallData::CapturedBatch::CancelWith(grpc_error_handle error,
                                              Flusher* releaser) {
-  auto* batch = absl::exchange(batch_, nullptr);
+  auto* batch = std::exchange(batch_, nullptr);
   GPR_ASSERT(batch != nullptr);
   uintptr_t& refcnt = *RefCountField(batch);
   if (refcnt == 0) {
@@ -282,8 +281,8 @@ class ClientCallData::PollContext {
             self_->recv_initial_metadata_->state =
                 RecvInitialMetadata::kResponded;
             flusher_->AddClosure(
-                absl::exchange(self_->recv_initial_metadata_->original_on_ready,
-                               nullptr),
+                std::exchange(self_->recv_initial_metadata_->original_on_ready,
+                              nullptr),
                 GRPC_ERROR_NONE,
                 "wake_inside_combiner:recv_initial_metadata_ready");
           }
@@ -310,8 +309,8 @@ class ClientCallData::PollContext {
             }
             self_->recv_trailing_state_ = RecvTrailingState::kResponded;
             flusher_->AddClosure(
-                absl::exchange(self_->original_recv_trailing_metadata_ready_,
-                               nullptr),
+                std::exchange(self_->original_recv_trailing_metadata_ready_,
+                              nullptr),
                 GRPC_ERROR_NONE, "wake_inside_combiner:recv_trailing_ready:1");
             if (self_->recv_initial_metadata_ != nullptr) {
               switch (self_->recv_initial_metadata_->state) {
@@ -334,7 +333,7 @@ class ClientCallData::PollContext {
                   self_->recv_initial_metadata_->state =
                       RecvInitialMetadata::kResponded;
                   flusher_->AddClosure(
-                      absl::exchange(
+                      std::exchange(
                           self_->recv_initial_metadata_->original_on_ready,
                           nullptr),
                       GRPC_ERROR_CANCELLED,
@@ -376,7 +375,7 @@ class ClientCallData::PollContext {
                   self_->recv_initial_metadata_->state =
                       RecvInitialMetadata::kResponded;
                   flusher_->AddClosure(
-                      absl::exchange(
+                      std::exchange(
                           self_->recv_initial_metadata_->original_on_ready,
                           nullptr),
                       GRPC_ERROR_REF(error),
@@ -419,8 +418,8 @@ class ClientCallData::PollContext {
         if (self_->recv_trailing_state_ == RecvTrailingState::kComplete) {
           self_->recv_trailing_state_ = RecvTrailingState::kResponded;
           flusher_->AddClosure(
-              absl::exchange(self_->original_recv_trailing_metadata_ready_,
-                             nullptr),
+              std::exchange(self_->original_recv_trailing_metadata_ready_,
+                            nullptr),
               GRPC_ERROR_NONE, "wake_inside_combiner:recv_trailing_ready:2");
         }
         break;
@@ -646,7 +645,7 @@ void ClientCallData::Cancel(grpc_error_handle error) {
         recv_initial_metadata_->state = RecvInitialMetadata::kResponded;
         GRPC_CALL_COMBINER_START(
             call_combiner(),
-            absl::exchange(recv_initial_metadata_->original_on_ready, nullptr),
+            std::exchange(recv_initial_metadata_->original_on_ready, nullptr),
             GRPC_ERROR_REF(error), "propagate cancellation");
         break;
       case RecvInitialMetadata::kInitial:
@@ -701,13 +700,13 @@ void ClientCallData::RecvInitialMetadataReady(grpc_error_handle error) {
   if (!GRPC_ERROR_IS_NONE(error)) {
     recv_initial_metadata_->state = RecvInitialMetadata::kResponded;
     flusher.AddClosure(
-        absl::exchange(recv_initial_metadata_->original_on_ready, nullptr),
+        std::exchange(recv_initial_metadata_->original_on_ready, nullptr),
         GRPC_ERROR_REF(error), "propagate cancellation");
   } else if (send_initial_state_ == SendInitialState::kCancelled ||
              recv_trailing_state_ == RecvTrailingState::kResponded) {
     recv_initial_metadata_->state = RecvInitialMetadata::kResponded;
     flusher.AddClosure(
-        absl::exchange(recv_initial_metadata_->original_on_ready, nullptr),
+        std::exchange(recv_initial_metadata_->original_on_ready, nullptr),
         GRPC_ERROR_REF(cancelled_error_), "propagate cancellation");
   }
   WakeInsideCombiner(&flusher);
@@ -827,7 +826,7 @@ void ClientCallData::RecvTrailingMetadataReady(grpc_error_handle error) {
   // forward the callback up with the same error.
   if (recv_trailing_state_ == RecvTrailingState::kCancelled) {
     if (grpc_closure* call_closure =
-            absl::exchange(original_recv_trailing_metadata_ready_, nullptr)) {
+            std::exchange(original_recv_trailing_metadata_ready_, nullptr)) {
       flusher.AddClosure(call_closure, GRPC_ERROR_REF(error),
                          "propagate failure");
     }
@@ -1080,7 +1079,7 @@ void ServerCallData::Cancel(grpc_error_handle error, Flusher* flusher) {
     send_initial_metadata_->state = SendInitialMetadata::kCancelled;
   }
   if (auto* closure =
-          absl::exchange(original_recv_initial_metadata_ready_, nullptr)) {
+          std::exchange(original_recv_initial_metadata_ready_, nullptr)) {
     flusher->AddClosure(closure, GRPC_ERROR_REF(error),
                         "original_recv_initial_metadata");
   }
@@ -1158,7 +1157,7 @@ void ServerCallData::RecvInitialMetadataReady(grpc_error_handle error) {
   if (!GRPC_ERROR_IS_NONE(error)) {
     recv_initial_state_ = RecvInitialState::kResponded;
     flusher.AddClosure(
-        absl::exchange(original_recv_initial_metadata_ready_, nullptr),
+        std::exchange(original_recv_initial_metadata_ready_, nullptr),
         GRPC_ERROR_REF(error), "propagate error");
     return;
   }
@@ -1178,7 +1177,7 @@ void ServerCallData::RecvInitialMetadataReady(grpc_error_handle error) {
   // Poll once.
   WakeInsideCombiner(&flusher);
   if (auto* closure =
-          absl::exchange(original_recv_initial_metadata_ready_, nullptr)) {
+          std::exchange(original_recv_initial_metadata_ready_, nullptr)) {
     flusher.AddClosure(closure, GRPC_ERROR_NONE,
                        "original_recv_initial_metadata");
   }

data/src/core/lib/channel/promise_based_filter.h

@@ -26,7 +26,6 @@
 
 #include <atomic>
 #include <new>
-#include <type_traits>
 #include <utility>
 
 #include "absl/container/inlined_vector.h"

data/src/core/lib/channel/status_util.cc

@@ -22,6 +22,8 @@
 
 #include <string.h>
 
+#include "absl/strings/str_cat.h"
+
 #include "src/core/lib/gpr/useful.h"
 
 struct status_string_entry {
@@ -109,3 +111,28 @@ bool grpc_status_code_from_int(int status_int, grpc_status_code* status) {
   *status = static_cast<grpc_status_code>(status_int);
   return true;
 }
+
+namespace grpc_core {
+
+absl::Status MaybeRewriteIllegalStatusCode(absl::Status status,
+                                           absl::string_view source) {
+  switch (status.code()) {
+    // The set of disallowed codes, as per
+    // https://github.com/grpc/proposal/blob/master/A54-restrict-control-plane-status-codes.md.
+    case absl::StatusCode::kInvalidArgument:
+    case absl::StatusCode::kNotFound:
+    case absl::StatusCode::kAlreadyExists:
+    case absl::StatusCode::kFailedPrecondition:
+    case absl::StatusCode::kAborted:
+    case absl::StatusCode::kOutOfRange:
+    case absl::StatusCode::kDataLoss: {
+      return absl::InternalError(
+          absl::StrCat("Illegal status code from ", source,
+                       "; original status: ", status.ToString()));
+    }
+    default:
+      return status;
+  }
+}
+
+}  // namespace grpc_core

data/src/core/lib/channel/status_util.h

@@ -21,6 +21,9 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "absl/status/status.h"
+#include "absl/strings/string_view.h"
+
 #include <grpc/status.h>
 
 /// If \a status_str is a valid status string, sets \a status to the
@@ -59,6 +62,13 @@ class StatusCodeSet {
 };
 
 }  // namespace internal
+
+// Optionally rewrites a status as per
+// https://github.com/grpc/proposal/blob/master/A54-restrict-control-plane-status-codes.md.
+// The source parameter indicates where the status came from.
+absl::Status MaybeRewriteIllegalStatusCode(absl::Status status,
+                                           absl::string_view source);
+
 }  // namespace grpc_core
 
 #endif /* GRPC_CORE_LIB_CHANNEL_STATUS_UTIL_H */

data/src/core/lib/config/core_configuration.cc

@@ -43,7 +43,11 @@ CoreConfiguration::CoreConfiguration(Builder* builder)
       handshaker_registry_(builder->handshaker_registry_.Build()),
       channel_creds_registry_(builder->channel_creds_registry_.Build()),
       service_config_parser_(builder->service_config_parser_.Build()),
-      resolver_registry_(builder->resolver_registry_.Build()) {}
+      resolver_registry_(builder->resolver_registry_.Build()),
+      lb_policy_registry_(builder->lb_policy_registry_.Build()),
+      proxy_mapper_registry_(builder->proxy_mapper_registry_.Build()),
+      certificate_provider_registry_(
+          builder->certificate_provider_registry_.Build()) {}
 
 void CoreConfiguration::RegisterBuilder(std::function<void(Builder*)> builder) {
   GPR_ASSERT(config_.load(std::memory_order_relaxed) == nullptr &&

data/src/core/lib/config/core_configuration.h

@@ -20,8 +20,13 @@
 #include <atomic>
 #include <functional>
 
+#include <grpc/support/log.h>
+
 #include "src/core/lib/channel/channel_args_preconditioning.h"
+#include "src/core/lib/handshaker/proxy_mapper_registry.h"
+#include "src/core/lib/load_balancing/lb_policy_registry.h"
 #include "src/core/lib/resolver/resolver_registry.h"
+#include "src/core/lib/security/certificate_provider/certificate_provider_registry.h"
 #include "src/core/lib/security/credentials/channel_creds_registry.h"
 #include "src/core/lib/service_config/service_config_parser.h"
 #include "src/core/lib/surface/channel_init.h"
@@ -62,6 +67,18 @@ class CoreConfiguration {
       return &resolver_registry_;
     }
 
+    LoadBalancingPolicyRegistry::Builder* lb_policy_registry() {
+      return &lb_policy_registry_;
+    }
+
+    ProxyMapperRegistry::Builder* proxy_mapper_registry() {
+      return &proxy_mapper_registry_;
+    }
+
+    CertificateProviderRegistry::Builder* certificate_provider_registry() {
+      return &certificate_provider_registry_;
+    }
+
    private:
     friend class CoreConfiguration;
 
@@ -71,11 +88,58 @@ class CoreConfiguration {
     ChannelCredsRegistry<>::Builder channel_creds_registry_;
     ServiceConfigParser::Builder service_config_parser_;
     ResolverRegistry::Builder resolver_registry_;
+    LoadBalancingPolicyRegistry::Builder lb_policy_registry_;
+    ProxyMapperRegistry::Builder proxy_mapper_registry_;
+    CertificateProviderRegistry::Builder certificate_provider_registry_;
 
     Builder();
     CoreConfiguration* Build();
   };
 
+  // Stores a builder for RegisterBuilder
+  struct RegisteredBuilder {
+    std::function<void(Builder*)> builder;
+    RegisteredBuilder* next;
+  };
+
+  // Temporarily replaces core configuration with what is built from the
+  // provided BuildFunc that takes (Builder*) and returns void.
+  // Requires no concurrent Get() be called. Restores current core
+  // configuration when this object is destroyed. The default builder
+  // is not backed up or restored.
+  //
+  // Useful for running multiple tests back to back in the same process
+  // without side effects from previous tests.
+  class WithSubstituteBuilder {
+   public:
+    template <typename BuildFunc>
+    explicit WithSubstituteBuilder(BuildFunc build) {
+      // Build core configuration to replace.
+      Builder builder;
+      build(&builder);
+      CoreConfiguration* p = builder.Build();
+
+      // Backup current core configuration and replace/reset.
+      config_restore_ =
+          CoreConfiguration::config_.exchange(p, std::memory_order_acquire);
+      builders_restore_ = CoreConfiguration::builders_.exchange(
+          nullptr, std::memory_order_acquire);
+    }
+
+    ~WithSubstituteBuilder() {
+      // Reset and restore.
+      Reset();
+      GPR_ASSERT(CoreConfiguration::config_.exchange(
+                     config_restore_, std::memory_order_acquire) == nullptr);
+      GPR_ASSERT(CoreConfiguration::builders_.exchange(
+                     builders_restore_, std::memory_order_acquire) == nullptr);
+    }
+
+   private:
+    CoreConfiguration* config_restore_;
+    RegisteredBuilder* builders_restore_;
+  };
+
   // Lifetime methods
 
   // Get the core configuration; if it does not exist, create it.
@@ -87,36 +151,11 @@ class CoreConfiguration {
     return BuildNewAndMaybeSet();
   }
 
-  // Build a special core configuration.
-  // Requires no concurrent Get() be called.
-  // Doesn't call the regular BuildCoreConfiguration function, instead calls
-  // `build`.
-  // BuildFunc is a callable that takes a Builder* and returns void.
-  // We use a template instead of std::function<void(Builder*)> to avoid
-  // including std::function in this widely used header, and to ensure no code
-  // is generated in programs that do not use this function.
-  // This is sometimes useful for testing.
-  template <typename BuildFunc>
-  static void BuildSpecialConfiguration(BuildFunc build) {
-    // Build bespoke configuration
-    Builder builder;
-    build(&builder);
-    CoreConfiguration* p = builder.Build();
-    // Swap in final configuration, deleting anything that was already present.
-    delete config_.exchange(p, std::memory_order_release);
-  }
-
   // Attach a registration function globally.
   // Each registration function is called *in addition to*
-  // BuildCoreConfiguration for the default core configuration. When using
-  // BuildSpecialConfiguration, one can use CallRegisteredBuilders to call them.
-  // Must be called before a configuration is built.
+  // BuildCoreConfiguration for the default core configuration.
   static void RegisterBuilder(std::function<void(Builder*)> builder);
 
-  // Call all registered builders.
-  // See RegisterBuilder for why you might want to call this.
-  static void CallRegisteredBuilders(Builder* builder);
-
   // Drop the core configuration. Users must ensure no other threads are
   // accessing the configuration.
   // Clears any dynamically registered builders.
@@ -128,10 +167,8 @@ class CoreConfiguration {
   template <typename BuildFunc, typename RunFunc>
   static void RunWithSpecialConfiguration(BuildFunc build_configuration,
                                           RunFunc code_to_run) {
-    Reset();
-    BuildSpecialConfiguration(build_configuration);
+    WithSubstituteBuilder builder(build_configuration);
     code_to_run();
-    Reset();
   }
 
   // Accessors
@@ -158,6 +195,18 @@ class CoreConfiguration {
     return resolver_registry_;
   }
 
+  const LoadBalancingPolicyRegistry& lb_policy_registry() const {
+    return lb_policy_registry_;
+  }
+
+  const ProxyMapperRegistry& proxy_mapper_registry() const {
+    return proxy_mapper_registry_;
+  }
+
+  const CertificateProviderRegistry& certificate_provider_registry() const {
+    return certificate_provider_registry_;
+  }
+
   static void SetDefaultBuilder(void (*builder)(CoreConfiguration::Builder*)) {
     default_builder_ = builder;
   }
@@ -165,12 +214,6 @@ class CoreConfiguration {
  private:
   explicit CoreConfiguration(Builder* builder);
 
-  // Stores a builder for RegisterBuilder
-  struct RegisteredBuilder {
-    std::function<void(Builder*)> builder;
-    RegisteredBuilder* next;
-  };
-
   // Create a new CoreConfiguration, and either set it or throw it away.
   // We allow multiple CoreConfiguration's to be created in parallel.
   static const CoreConfiguration& BuildNewAndMaybeSet();
@@ -188,6 +231,9 @@ class CoreConfiguration {
   ChannelCredsRegistry<> channel_creds_registry_;
   ServiceConfigParser service_config_parser_;
   ResolverRegistry resolver_registry_;
+  LoadBalancingPolicyRegistry lb_policy_registry_;
+  ProxyMapperRegistry proxy_mapper_registry_;
+  CertificateProviderRegistry certificate_provider_registry_;
 };
 
 extern void BuildCoreConfiguration(CoreConfiguration::Builder* builder);

data/src/core/lib/debug/stats.cc

@@ -26,35 +26,33 @@
 #include <algorithm>
 #include <vector>
 
+#include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/cpu.h>
-#include <grpc/support/sync.h>
-
-grpc_stats_data* grpc_stats_per_cpu_storage = nullptr;
-static size_t g_num_cores;
-static gpr_once g_once = GPR_ONCE_INIT;
-
-void grpc_stats_init(void) {
-  gpr_once_init(&g_once, []() {
-    g_num_cores = gpr_cpu_num_cores();
-    grpc_stats_per_cpu_storage = static_cast<grpc_stats_data*>(
-        gpr_zalloc(sizeof(grpc_stats_data) * g_num_cores));
-  });
-}
+
+namespace grpc_core {
+Stats* const g_stats_data = [] {
+  size_t num_cores = gpr_cpu_num_cores();
+  Stats* stats = static_cast<Stats*>(
+      gpr_zalloc(sizeof(Stats) + num_cores * sizeof(grpc_stats_data)));
+  stats->num_cores = num_cores;
+  return stats;
+}();
+}  // namespace grpc_core
 
 void grpc_stats_collect(grpc_stats_data* output) {
   memset(output, 0, sizeof(*output));
-  for (size_t core = 0; core < g_num_cores; core++) {
+  for (size_t core = 0; core < grpc_core::g_stats_data->num_cores; core++) {
     for (size_t i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
       output->counters[i] += gpr_atm_no_barrier_load(
-          &grpc_stats_per_cpu_storage[core].counters[i]);
+          &grpc_core::g_stats_data->per_cpu[core].counters[i]);
     }
     for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_BUCKETS; i++) {
       output->histograms[i] += gpr_atm_no_barrier_load(
-          &grpc_stats_per_cpu_storage[core].histograms[i]);
+          &grpc_core::g_stats_data->per_cpu[core].histograms[i]);
     }
   }
 }
@@ -69,21 +67,12 @@ void grpc_stats_diff(const grpc_stats_data* b, const grpc_stats_data* a,
   }
 }
 
-int grpc_stats_histo_find_bucket_slow(int value, const int* table,
-                                      int table_size) {
-  GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS();
-  const int* const start = table;
-  while (table_size > 0) {
-    int step = table_size / 2;
-    const int* it = table + step;
-    if (value >= *it) {
-      table = it + 1;
-      table_size -= step + 1;
-    } else {
-      table_size = step;
-    }
-  }
-  return static_cast<int>(table - start) - 1;
+void grpc_stats_inc_histogram_value(int histogram, int value) {
+  const int bucket = grpc_stats_get_bucket[histogram](value);
+  gpr_atm_no_barrier_fetch_add(
+      &GRPC_THREAD_STATS_DATA()
+           ->histograms[grpc_stats_histo_start[histogram] + bucket],
+      1);
 }
 
 size_t grpc_stats_histo_count(const grpc_stats_data* stats,
@@ -145,28 +134,32 @@ double grpc_stats_histo_percentile(const grpc_stats_data* stats,
       static_cast<double>(count) * percentile / 100.0);
 }
 
+namespace {
+template <typename I>
+std::string ArrayToJson(const I* values, size_t count) {
+  std::vector<std::string> parts;
+  for (size_t i = 0; i < count; i++) {
+    parts.push_back(absl::StrFormat("%d", values[i]));
+  }
+  return absl::StrCat("[", absl::StrJoin(parts, ","), "]");
+}
+}  // namespace
+
 std::string grpc_stats_data_as_json(const grpc_stats_data* data) {
   std::vector<std::string> parts;
-  parts.push_back("{");
   for (size_t i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
     parts.push_back(absl::StrFormat(
         "\"%s\": %" PRIdPTR, grpc_stats_counter_name[i], data->counters[i]));
   }
   for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) {
-    parts.push_back(absl::StrFormat("\"%s\": [", grpc_stats_histogram_name[i]));
-    for (int j = 0; j < grpc_stats_histo_buckets[i]; j++) {
-      parts.push_back(
-          absl::StrFormat("%s%" PRIdPTR, j == 0 ? "" : ",",
-                          data->histograms[grpc_stats_histo_start[i] + j]));
-    }
+    parts.push_back(absl::StrFormat(
+        "\"%s\": %s", grpc_stats_histogram_name[i],
+        ArrayToJson(data->histograms + grpc_stats_histo_start[i],
+                    grpc_stats_histo_buckets[i])));
     parts.push_back(
-        absl::StrFormat("], \"%s_bkt\": [", grpc_stats_histogram_name[i]));
-    for (int j = 0; j < grpc_stats_histo_buckets[i]; j++) {
-      parts.push_back(absl::StrFormat(
-          "%s%d", j == 0 ? "" : ",", grpc_stats_histo_bucket_boundaries[i][j]));
-    }
-    parts.push_back("]");
+        absl::StrFormat("\"%s_bkt\": %s", grpc_stats_histogram_name[i],
+                        ArrayToJson(grpc_stats_histo_bucket_boundaries[i],
+                                    grpc_stats_histo_buckets[i])));
   }
-  parts.push_back("}");
-  return absl::StrJoin(parts, "");
+  return absl::StrCat("{", absl::StrJoin(parts, ", "), "}");
 }

data/src/core/lib/debug/stats.h

@@ -27,7 +27,7 @@
 
 #include <grpc/support/atm.h>
 
-#include "src/core/lib/debug/stats_data.h"
+#include "src/core/lib/debug/stats_data.h"  // IWYU pragma: export
 #include "src/core/lib/iomgr/exec_ctx.h"
 
 typedef struct grpc_stats_data {
@@ -35,14 +35,18 @@ typedef struct grpc_stats_data {
   gpr_atm histograms[GRPC_STATS_HISTOGRAM_BUCKETS];
 } grpc_stats_data;
 
-extern grpc_stats_data* grpc_stats_per_cpu_storage;
+namespace grpc_core {
+struct Stats {
+  size_t num_cores;
+  grpc_stats_data per_cpu[0];
+};
+extern Stats* const g_stats_data;
+}  // namespace grpc_core
 
 #define GRPC_THREAD_STATS_DATA() \
-  (&grpc_stats_per_cpu_storage[grpc_core::ExecCtx::Get()->starting_cpu()])
+  (&::grpc_core::g_stats_data    \
+        ->per_cpu[grpc_core::ExecCtx::Get()->starting_cpu()])
 
-/* Only collect stats if GRPC_COLLECT_STATS is defined or it is a debug build.
- */
-#if defined(GRPC_COLLECT_STATS) || !defined(NDEBUG)
 #define GRPC_STATS_INC_COUNTER(ctr) \
   (gpr_atm_no_barrier_fetch_add(&GRPC_THREAD_STATS_DATA()->counters[(ctr)], 1))
 
@@ -50,23 +54,17 @@ extern grpc_stats_data* grpc_stats_per_cpu_storage;
   (gpr_atm_no_barrier_fetch_add(                                               \
       &GRPC_THREAD_STATS_DATA()->histograms[histogram##_FIRST_SLOT + (index)], \
       1))
-#else /* defined(GRPC_COLLECT_STATS) || !defined(NDEBUG) */
-#define GRPC_STATS_INC_COUNTER(ctr)
-#define GRPC_STATS_INC_HISTOGRAM(histogram, index)
-#endif /* defined(GRPC_COLLECT_STATS) || !defined(NDEBUG) */
 
-void grpc_stats_init(void);
 void grpc_stats_collect(grpc_stats_data* output);
 // c = b-a
 void grpc_stats_diff(const grpc_stats_data* b, const grpc_stats_data* a,
                      grpc_stats_data* c);
 std::string grpc_stats_data_as_json(const grpc_stats_data* data);
-int grpc_stats_histo_find_bucket_slow(int value, const int* table,
-                                      int table_size);
 double grpc_stats_histo_percentile(const grpc_stats_data* stats,
                                    grpc_stats_histograms histogram,
                                    double percentile);
 size_t grpc_stats_histo_count(const grpc_stats_data* stats,
                               grpc_stats_histograms histogram);
+void grpc_stats_inc_histogram_value(int histogram, int value);
 
 #endif  // GRPC_CORE_LIB_DEBUG_STATS_H