grpc 1.50.0 → 1.51.0

data/src/core/lib/compression/message_compress.cc

@@ -29,7 +29,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/lib/slice/slice_refcount.h"
+#include "src/core/lib/slice/slice.h"
 
 #define OUTPUT_BLOCK_SIZE 1024
 
@@ -82,7 +82,7 @@ static int zlib_body(z_stream* zs, grpc_slice_buffer* input,
   return 1;
 
 error:
-  grpc_slice_unref_internal(outbuf);
+  grpc_core::CSliceUnref(outbuf);
   return 0;
 }
 
@@ -109,7 +109,7 @@ static int zlib_compress(grpc_slice_buffer* input, grpc_slice_buffer* output,
   r = zlib_body(&zs, input, output, deflate) && output->length < input->length;
   if (!r) {
     for (i = count_before; i < output->count; i++) {
-      grpc_slice_unref_internal(output->slices[i]);
+      grpc_core::CSliceUnref(output->slices[i]);
     }
     output->count = count_before;
     output->length = length_before;
@@ -133,7 +133,7 @@ static int zlib_decompress(grpc_slice_buffer* input, grpc_slice_buffer* output,
   r = zlib_body(&zs, input, output, inflate);
   if (!r) {
     for (i = count_before; i < output->count; i++) {
-      grpc_slice_unref_internal(output->slices[i]);
+      grpc_core::CSliceUnref(output->slices[i]);
     }
     output->count = count_before;
     output->length = length_before;
@@ -145,7 +145,7 @@ static int zlib_decompress(grpc_slice_buffer* input, grpc_slice_buffer* output,
 static int copy(grpc_slice_buffer* input, grpc_slice_buffer* output) {
   size_t i;
   for (i = 0; i < input->count; i++) {
-    grpc_slice_buffer_add(output, grpc_slice_ref_internal(input->slices[i]));
+    grpc_slice_buffer_add(output, grpc_core::CSliceRef(input->slices[i]));
   }
   return 1;
 }

data/src/core/lib/debug/event_log.cc

@@ -0,0 +1,88 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/debug/event_log.h"
+
+#include <algorithm>
+#include <atomic>
+
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
+
+#include <grpc/support/log.h>
+
+namespace grpc_core {
+
+std::atomic<EventLog*> EventLog::g_instance_{nullptr};
+
+EventLog::~EventLog() {
+  GPR_ASSERT(g_instance_.load(std::memory_order_acquire) != this);
+}
+
+void EventLog::BeginCollection() {
+  for (auto& fragment : fragments_) {
+    MutexLock lock(&fragment.mu);
+    fragment.entries.clear();
+  }
+  collection_begin_ = gpr_get_cycle_counter();
+  g_instance_.store(this, std::memory_order_release);
+  Append("logging", 1);
+}
+
+std::vector<EventLog::Entry> EventLog::EndCollection(
+    absl::Span<const absl::string_view> wanted_events) {
+  Append("logging", -1);
+  g_instance_.store(nullptr, std::memory_order_release);
+  std::vector<Entry> result;
+  for (auto& fragment : fragments_) {
+    MutexLock lock(&fragment.mu);
+    for (const auto& entry : fragment.entries) {
+      if (std::find(wanted_events.begin(), wanted_events.end(), entry.event) !=
+          wanted_events.end()) {
+        result.push_back(entry);
+      }
+    }
+    fragment.entries.clear();
+  }
+  std::stable_sort(
+      result.begin(), result.end(),
+      [](const Entry& a, const Entry& b) { return a.when < b.when; });
+  return result;
+}
+
+void EventLog::AppendInternal(absl::string_view event, int64_t delta) {
+  auto& fragment = fragments_.this_cpu();
+  MutexLock lock(&fragment.mu);
+  fragment.entries.push_back({gpr_get_cycle_counter(), event, delta});
+}
+
+std::string EventLog::EndCollectionAndReportCsv(
+    absl::Span<const absl::string_view> columns) {
+  auto events = EndCollection(columns);
+  std::vector<int64_t> values(columns.size(), 0);
+  std::string result =
+      absl::StrCat("timestamp,", absl::StrJoin(columns, ","), "\n");
+  for (const auto& entry : events) {
+    auto idx = std::find(columns.begin(), columns.end(), entry.event) -
+               columns.begin();
+    values[idx] += entry.delta;
+    absl::StrAppend(&result, entry.when - collection_begin_, ",",
+                    absl::StrJoin(values, ","), "\n");
+  }
+  return result;
+}
+
+}  // namespace grpc_core

data/src/core/lib/debug/event_log.h

@@ -0,0 +1,81 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_DEBUG_EVENT_LOG_H
+#define GRPC_CORE_LIB_DEBUG_EVENT_LOG_H
+
+#include <grpc/support/port_platform.h>
+
+#include <stdint.h>
+
+#include <atomic>
+#include <string>
+#include <vector>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/strings/string_view.h"
+#include "absl/types/span.h"
+
+#include "src/core/lib/gpr/time_precise.h"
+#include "src/core/lib/gprpp/per_cpu.h"
+#include "src/core/lib/gprpp/sync.h"
+
+namespace grpc_core {
+
+// Debug utility to collect a burst of events and then later log them as a
+// detailed sequence.
+// Collects (timestamp, counter-name, delta) and gives back a csv with
+// timestamps and accumulated values for each counter in separate columns.
+class EventLog {
+ public:
+  EventLog() = default;
+  ~EventLog();
+
+  EventLog(const EventLog&) = delete;
+  EventLog& operator=(const EventLog&) = delete;
+
+  void BeginCollection();
+  std::string EndCollectionAndReportCsv(
+      absl::Span<const absl::string_view> columns);
+
+  static void Append(absl::string_view event, int64_t delta) {
+    EventLog* log = g_instance_.load(std::memory_order_acquire);
+    if (log == nullptr) return;
+    log->AppendInternal(event, delta);
+  }
+
+ private:
+  struct Entry {
+    gpr_cycle_counter when;
+    absl::string_view event;
+    int64_t delta;
+  };
+
+  struct Fragment {
+    Mutex mu;
+    std::vector<Entry> entries ABSL_GUARDED_BY(mu);
+  };
+
+  void AppendInternal(absl::string_view event, int64_t delta);
+  std::vector<Entry> EndCollection(
+      absl::Span<const absl::string_view> wanted_events);
+
+  PerCpu<Fragment> fragments_;
+  gpr_cycle_counter collection_begin_;
+  static std::atomic<EventLog*> g_instance_;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_LIB_DEBUG_EVENT_LOG_H

data/src/core/lib/debug/histogram_view.cc

@@ -0,0 +1,69 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/debug/histogram_view.h"
+
+namespace grpc_core {
+
+double HistogramView::Count() const {
+  double sum = 0;
+  for (int i = 0; i < num_buckets; i++) {
+    sum += buckets[i];
+  }
+  return sum;
+}
+
+double HistogramView::ThresholdForCountBelow(double count_below) const {
+  double lower_bound;
+  double upper_bound;
+  int upper_idx;
+
+  // find the lowest bucket that gets us above count_below
+  double count_so_far = 0.0;
+  int lower_idx = 0;
+  for (; lower_idx < num_buckets; lower_idx++) {
+    count_so_far += static_cast<double>(buckets[lower_idx]);
+    if (count_so_far >= count_below) {
+      break;
+    }
+  }
+  if (count_so_far == count_below) {
+    // this bucket hits the threshold exactly... we should be midway through
+    // any run of zero values following the bucket
+    for (upper_idx = lower_idx + 1; upper_idx < num_buckets; upper_idx++) {
+      if (buckets[upper_idx]) {
+        break;
+      }
+    }
+    return (bucket_boundaries[lower_idx] + bucket_boundaries[upper_idx]) / 2.0;
+  } else {
+    // treat values as uniform throughout the bucket, and find where this value
+    // should lie
+    lower_bound = bucket_boundaries[lower_idx];
+    upper_bound = bucket_boundaries[lower_idx + 1];
+    return upper_bound - (upper_bound - lower_bound) *
+                             (count_so_far - count_below) /
+                             static_cast<double>(buckets[lower_idx]);
+  }
+}
+
+double HistogramView::Percentile(double p) const {
+  const double count = Count();
+  if (count == 0) return 0.0;
+  return ThresholdForCountBelow(count * p / 100.0);
+}
+
+}  // namespace grpc_core

data/src/core/lib/{slice/slice_refcount.cc → debug/histogram_view.h}

@@ -12,24 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <grpc/support/port_platform.h>
+#ifndef GRPC_CORE_LIB_DEBUG_HISTOGRAM_VIEW_H
+#define GRPC_CORE_LIB_DEBUG_HISTOGRAM_VIEW_H
 
-#include "src/core/lib/slice/slice_refcount.h"
+#include <grpc/support/port_platform.h>
 
-#include <chrono>
+#include <stdint.h>
 
 namespace grpc_core {
 
-uint32_t g_hash_seed = []() {
-  auto now = std::chrono::system_clock::now();
-  auto now_since_epoch = now.time_since_epoch();
-  auto now_ns =
-      std::chrono::duration_cast<std::chrono::nanoseconds>(now_since_epoch);
-  return static_cast<uint32_t>(now_ns.count());
-}();
+struct HistogramView {
+  int (*bucket_for)(int value);
+  const int* bucket_boundaries;
+  int num_buckets;
+  const uint64_t* buckets;
+
+  double Percentile(double p) const;
+  double Count() const;
+  double ThresholdForCountBelow(double count_below) const;
+};
 
 }  // namespace grpc_core
 
-void grpc_test_only_set_slice_hash_seed(uint32_t seed) {
-  grpc_core::g_hash_seed = seed;
-}
+#endif  // GRPC_CORE_LIB_DEBUG_HISTOGRAM_VIEW_H

data/src/core/lib/debug/stats.cc

@@ -20,146 +20,49 @@
 
 #include "src/core/lib/debug/stats.h"
 
-#include <inttypes.h>
-#include <string.h>
+#include <stddef.h>
 
 #include <algorithm>
 #include <vector>
 
 #include "absl/strings/str_cat.h"
-#include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 
-#include <grpc/support/alloc.h>
-#include <grpc/support/cpu.h>
-
 namespace grpc_core {
-Stats* const g_stats_data = [] {
-  size_t num_cores = gpr_cpu_num_cores();
-  Stats* stats = static_cast<Stats*>(
-      gpr_zalloc(sizeof(Stats) + num_cores * sizeof(grpc_stats_data)));
-  stats->num_cores = num_cores;
-  return stats;
-}();
-}  // namespace grpc_core
-
-void grpc_stats_collect(grpc_stats_data* output) {
-  memset(output, 0, sizeof(*output));
-  for (size_t core = 0; core < grpc_core::g_stats_data->num_cores; core++) {
-    for (size_t i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
-      output->counters[i] += gpr_atm_no_barrier_load(
-          &grpc_core::g_stats_data->per_cpu[core].counters[i]);
-    }
-    for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_BUCKETS; i++) {
-      output->histograms[i] += gpr_atm_no_barrier_load(
-          &grpc_core::g_stats_data->per_cpu[core].histograms[i]);
-    }
-  }
-}
-
-void grpc_stats_diff(const grpc_stats_data* b, const grpc_stats_data* a,
-                     grpc_stats_data* c) {
-  for (size_t i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
-    c->counters[i] = b->counters[i] - a->counters[i];
-  }
-  for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_BUCKETS; i++) {
-    c->histograms[i] = b->histograms[i] - a->histograms[i];
-  }
-}
-
-void grpc_stats_inc_histogram_value(int histogram, int value) {
-  const int bucket = grpc_stats_get_bucket[histogram](value);
-  gpr_atm_no_barrier_fetch_add(
-      &GRPC_THREAD_STATS_DATA()
-           ->histograms[grpc_stats_histo_start[histogram] + bucket],
-      1);
-}
 
-size_t grpc_stats_histo_count(const grpc_stats_data* stats,
-                              grpc_stats_histograms histogram) {
-  size_t sum = 0;
-  for (int i = 0; i < grpc_stats_histo_buckets[histogram]; i++) {
-    sum += static_cast<size_t>(
-        stats->histograms[grpc_stats_histo_start[histogram] + i]);
-  }
-  return sum;
-}
-
-static double threshold_for_count_below(const gpr_atm* bucket_counts,
-                                        const int* bucket_boundaries,
-                                        int num_buckets, double count_below) {
-  double count_so_far;
-  double lower_bound;
-  double upper_bound;
-  int lower_idx;
-  int upper_idx;
-
-  /* find the lowest bucket that gets us above count_below */
-  count_so_far = 0.0;
-  for (lower_idx = 0; lower_idx < num_buckets; lower_idx++) {
-    count_so_far += static_cast<double>(bucket_counts[lower_idx]);
-    if (count_so_far >= count_below) {
-      break;
-    }
-  }
-  if (count_so_far == count_below) {
-    /* this bucket hits the threshold exactly... we should be midway through
-       any run of zero values following the bucket */
-    for (upper_idx = lower_idx + 1; upper_idx < num_buckets; upper_idx++) {
-      if (bucket_counts[upper_idx]) {
-        break;
-      }
-    }
-    return (bucket_boundaries[lower_idx] + bucket_boundaries[upper_idx]) / 2.0;
-  } else {
-    /* treat values as uniform throughout the bucket, and find where this value
-       should lie */
-    lower_bound = bucket_boundaries[lower_idx];
-    upper_bound = bucket_boundaries[lower_idx + 1];
-    return upper_bound - (upper_bound - lower_bound) *
-                             (count_so_far - count_below) /
-                             static_cast<double>(bucket_counts[lower_idx]);
-  }
-}
-
-double grpc_stats_histo_percentile(const grpc_stats_data* stats,
-                                   grpc_stats_histograms histogram,
-                                   double percentile) {
-  size_t count = grpc_stats_histo_count(stats, histogram);
-  if (count == 0) return 0.0;
-  return threshold_for_count_below(
-      stats->histograms + grpc_stats_histo_start[histogram],
-      grpc_stats_histo_bucket_boundaries[histogram],
-      grpc_stats_histo_buckets[histogram],
-      static_cast<double>(count) * percentile / 100.0);
-}
+namespace stats_detail {
 
 namespace {
 template <typename I>
-std::string ArrayToJson(const I* values, size_t count) {
+std::string ArrayToJson(absl::Span<const I> values) {
   std::vector<std::string> parts;
-  for (size_t i = 0; i < count; i++) {
-    parts.push_back(absl::StrFormat("%d", values[i]));
+  for (auto value : values) {
+    parts.push_back(absl::StrCat(value));
   }
   return absl::StrCat("[", absl::StrJoin(parts, ","), "]");
 }
 }  // namespace
 
-std::string grpc_stats_data_as_json(const grpc_stats_data* data) {
+std::string StatsAsJson(absl::Span<const uint64_t> counters,
+                        absl::Span<const absl::string_view> counter_name,
+                        absl::Span<const HistogramView> histograms,
+                        absl::Span<const absl::string_view> histogram_name) {
   std::vector<std::string> parts;
-  for (size_t i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
-    parts.push_back(absl::StrFormat(
-        "\"%s\": %" PRIdPTR, grpc_stats_counter_name[i], data->counters[i]));
+  for (size_t i = 0; i < counters.size(); i++) {
+    parts.push_back(absl::StrCat("\"", counter_name[i], "\": ", counters[i]));
   }
-  for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) {
-    parts.push_back(absl::StrFormat(
-        "\"%s\": %s", grpc_stats_histogram_name[i],
-        ArrayToJson(data->histograms + grpc_stats_histo_start[i],
-                    grpc_stats_histo_buckets[i])));
+  for (size_t i = 0; i < histograms.size(); i++) {
     parts.push_back(
-        absl::StrFormat("\"%s_bkt\": %s", grpc_stats_histogram_name[i],
-                        ArrayToJson(grpc_stats_histo_bucket_boundaries[i],
-                                    grpc_stats_histo_buckets[i])));
+        absl::StrCat("\"", histogram_name[i], "\": ",
+                     ArrayToJson(absl::Span<const uint64_t>(
+                         histograms[i].buckets, histograms[i].num_buckets))));
+    parts.push_back(absl::StrCat(
+        "\"", histogram_name[i], "_bkt\": ",
+        ArrayToJson(absl::Span<const int>(histograms[i].bucket_boundaries,
+                                          histograms[i].num_buckets))));
   }
   return absl::StrCat("{", absl::StrJoin(parts, ", "), "}");
 }
+
+}  // namespace stats_detail
+}  // namespace grpc_core

data/src/core/lib/debug/stats.h

@@ -21,50 +21,44 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <stddef.h>
+#include <stdint.h>
 
 #include <string>
+#include <vector>
 
-#include <grpc/support/atm.h>
+#include "absl/strings/string_view.h"
+#include "absl/types/span.h"
 
-#include "src/core/lib/debug/stats_data.h"  // IWYU pragma: export
-#include "src/core/lib/iomgr/exec_ctx.h"
-
-typedef struct grpc_stats_data {
-  gpr_atm counters[GRPC_STATS_COUNTER_COUNT];
-  gpr_atm histograms[GRPC_STATS_HISTOGRAM_BUCKETS];
-} grpc_stats_data;
+#include "src/core/lib/debug/histogram_view.h"
+#include "src/core/lib/debug/stats_data.h"
+#include "src/core/lib/gprpp/no_destruct.h"
 
 namespace grpc_core {
-struct Stats {
-  size_t num_cores;
-  grpc_stats_data per_cpu[0];
-};
-extern Stats* const g_stats_data;
-}  // namespace grpc_core
 
-#define GRPC_THREAD_STATS_DATA() \
-  (&::grpc_core::g_stats_data    \
-        ->per_cpu[grpc_core::ExecCtx::Get()->starting_cpu()])
+inline GlobalStatsCollector& global_stats() {
+  return *NoDestructSingleton<GlobalStatsCollector>::Get();
+}
 
-#define GRPC_STATS_INC_COUNTER(ctr) \
-  (gpr_atm_no_barrier_fetch_add(&GRPC_THREAD_STATS_DATA()->counters[(ctr)], 1))
+namespace stats_detail {
+std::string StatsAsJson(absl::Span<const uint64_t> counters,
+                        absl::Span<const absl::string_view> counter_name,
+                        absl::Span<const HistogramView> histograms,
+                        absl::Span<const absl::string_view> histogram_name);
+}
 
-#define GRPC_STATS_INC_HISTOGRAM(histogram, index)                             \
-  (gpr_atm_no_barrier_fetch_add(                                               \
-      &GRPC_THREAD_STATS_DATA()->histograms[histogram##_FIRST_SLOT + (index)], \
-      1))
+template <typename T>
+std::string StatsAsJson(T* data) {
+  std::vector<HistogramView> histograms;
+  for (int i = 0; i < static_cast<int>(T::Histogram::COUNT); i++) {
+    histograms.push_back(
+        data->histogram(static_cast<typename T::Histogram>(i)));
+  }
+  return stats_detail::StatsAsJson(
+      absl::Span<const uint64_t>(data->counters,
+                                 static_cast<int>(T::Counter::COUNT)),
+      T::counter_name, histograms, T::histogram_name);
+}
 
-void grpc_stats_collect(grpc_stats_data* output);
-// c = b-a
-void grpc_stats_diff(const grpc_stats_data* b, const grpc_stats_data* a,
-                     grpc_stats_data* c);
-std::string grpc_stats_data_as_json(const grpc_stats_data* data);
-double grpc_stats_histo_percentile(const grpc_stats_data* stats,
-                                   grpc_stats_histograms histogram,
-                                   double percentile);
-size_t grpc_stats_histo_count(const grpc_stats_data* stats,
-                              grpc_stats_histograms histogram);
-void grpc_stats_inc_histogram_value(int histogram, int value);
+}  // namespace grpc_core
 
 #endif  // GRPC_CORE_LIB_DEBUG_STATS_H