grpc 1.42.0 → 1.45.0

data/third_party/abseil-cpp/absl/numeric/int128_no_intrinsic.inc

@@ -134,10 +134,6 @@ constexpr int128::operator unsigned long long() const {  // NOLINT(runtime/int)
   return static_cast<unsigned long long>(lo_);           // NOLINT(runtime/int)
 }
 
-// Forward declaration for conversion operators to floating point types.
-int128 operator-(int128 v);
-bool operator!=(int128 lhs, int128 rhs);
-
 inline int128::operator float() const {
   // We must convert the absolute value and then negate as needed, because
   // floating point types are typically sign-magnitude. Otherwise, the
@@ -169,76 +165,80 @@ inline int128::operator long double() const {
 
 // Comparison operators.
 
-inline bool operator==(int128 lhs, int128 rhs) {
+constexpr bool operator==(int128 lhs, int128 rhs) {
   return (Int128Low64(lhs) == Int128Low64(rhs) &&
           Int128High64(lhs) == Int128High64(rhs));
 }
 
-inline bool operator!=(int128 lhs, int128 rhs) {
-  return !(lhs == rhs);
-}
+constexpr bool operator!=(int128 lhs, int128 rhs) { return !(lhs == rhs); }
 
-inline bool operator<(int128 lhs, int128 rhs) {
+constexpr bool operator<(int128 lhs, int128 rhs) {
   return (Int128High64(lhs) == Int128High64(rhs))
              ? (Int128Low64(lhs) < Int128Low64(rhs))
              : (Int128High64(lhs) < Int128High64(rhs));
 }
 
-inline bool operator>(int128 lhs, int128 rhs) {
+constexpr bool operator>(int128 lhs, int128 rhs) {
   return (Int128High64(lhs) == Int128High64(rhs))
              ? (Int128Low64(lhs) > Int128Low64(rhs))
              : (Int128High64(lhs) > Int128High64(rhs));
 }
 
-inline bool operator<=(int128 lhs, int128 rhs) {
-  return !(lhs > rhs);
-}
+constexpr bool operator<=(int128 lhs, int128 rhs) { return !(lhs > rhs); }
 
-inline bool operator>=(int128 lhs, int128 rhs) {
-  return !(lhs < rhs);
-}
+constexpr bool operator>=(int128 lhs, int128 rhs) { return !(lhs < rhs); }
 
 // Unary operators.
 
-inline int128 operator-(int128 v) {
-  int64_t hi = ~Int128High64(v);
-  uint64_t lo = ~Int128Low64(v) + 1;
-  if (lo == 0) ++hi;  // carry
-  return MakeInt128(hi, lo);
+constexpr int128 operator-(int128 v) {
+  return MakeInt128(~Int128High64(v) + (Int128Low64(v) == 0),
+                    ~Int128Low64(v) + 1);
 }
 
-inline bool operator!(int128 v) {
+constexpr bool operator!(int128 v) {
   return !Int128Low64(v) && !Int128High64(v);
 }
 
-inline int128 operator~(int128 val) {
+constexpr int128 operator~(int128 val) {
   return MakeInt128(~Int128High64(val), ~Int128Low64(val));
 }
 
 // Arithmetic operators.
 
-inline int128 operator+(int128 lhs, int128 rhs) {
-  int128 result = MakeInt128(Int128High64(lhs) + Int128High64(rhs),
-                             Int128Low64(lhs) + Int128Low64(rhs));
-  if (Int128Low64(result) < Int128Low64(lhs)) {  // check for carry
-    return MakeInt128(Int128High64(result) + 1, Int128Low64(result));
-  }
-  return result;
+namespace int128_internal {
+constexpr int128 SignedAddResult(int128 result, int128 lhs) {
+  // check for carry
+  return (Int128Low64(result) < Int128Low64(lhs))
+             ? MakeInt128(Int128High64(result) + 1, Int128Low64(result))
+             : result;
+}
+}  // namespace int128_internal
+constexpr int128 operator+(int128 lhs, int128 rhs) {
+  return int128_internal::SignedAddResult(
+      MakeInt128(Int128High64(lhs) + Int128High64(rhs),
+                 Int128Low64(lhs) + Int128Low64(rhs)),
+      lhs);
 }
 
-inline int128 operator-(int128 lhs, int128 rhs) {
-  int128 result = MakeInt128(Int128High64(lhs) - Int128High64(rhs),
-                             Int128Low64(lhs) - Int128Low64(rhs));
-  if (Int128Low64(lhs) < Int128Low64(rhs)) {  // check for carry
-    return MakeInt128(Int128High64(result) - 1, Int128Low64(result));
-  }
-  return result;
+namespace int128_internal {
+constexpr int128 SignedSubstructResult(int128 result, int128 lhs, int128 rhs) {
+  // check for carry
+  return (Int128Low64(lhs) < Int128Low64(rhs))
+             ? MakeInt128(Int128High64(result) - 1, Int128Low64(result))
+             : result;
+}
+}  // namespace int128_internal
+constexpr int128 operator-(int128 lhs, int128 rhs) {
+  return int128_internal::SignedSubstructResult(
+      MakeInt128(Int128High64(lhs) - Int128High64(rhs),
+                 Int128Low64(lhs) - Int128Low64(rhs)),
+      lhs, rhs);
 }
 
 inline int128 operator*(int128 lhs, int128 rhs) {
-  uint128 result = uint128(lhs) * rhs;
-  return MakeInt128(int128_internal::BitCastToSigned(Uint128High64(result)),
-                    Uint128Low64(result));
+  return MakeInt128(
+      int128_internal::BitCastToSigned(Uint128High64(uint128(lhs) * rhs)),
+      Uint128Low64(uint128(lhs) * rhs));
 }
 
 inline int128 int128::operator++(int) {
@@ -263,46 +263,49 @@ inline int128& int128::operator--() {
   return *this;
 }
 
-inline int128 operator|(int128 lhs, int128 rhs) {
+constexpr int128 operator|(int128 lhs, int128 rhs) {
   return MakeInt128(Int128High64(lhs) | Int128High64(rhs),
                     Int128Low64(lhs) | Int128Low64(rhs));
 }
 
-inline int128 operator&(int128 lhs, int128 rhs) {
+constexpr int128 operator&(int128 lhs, int128 rhs) {
   return MakeInt128(Int128High64(lhs) & Int128High64(rhs),
                     Int128Low64(lhs) & Int128Low64(rhs));
 }
 
-inline int128 operator^(int128 lhs, int128 rhs) {
+constexpr int128 operator^(int128 lhs, int128 rhs) {
   return MakeInt128(Int128High64(lhs) ^ Int128High64(rhs),
                     Int128Low64(lhs) ^ Int128Low64(rhs));
 }
 
-inline int128 operator<<(int128 lhs, int amount) {
-  // uint64_t shifts of >= 64 are undefined, so we need some special-casing.
-  if (amount < 64) {
-    if (amount != 0) {
-      return MakeInt128(
-          (Int128High64(lhs) << amount) |
-              static_cast<int64_t>(Int128Low64(lhs) >> (64 - amount)),
-          Int128Low64(lhs) << amount);
-    }
-    return lhs;
-  }
-  return MakeInt128(static_cast<int64_t>(Int128Low64(lhs) << (amount - 64)), 0);
-}
-
-inline int128 operator>>(int128 lhs, int amount) {
-  // uint64_t shifts of >= 64 are undefined, so we need some special-casing.
-  if (amount < 64) {
-    if (amount != 0) {
-      return MakeInt128(
-          Int128High64(lhs) >> amount,
-          (Int128Low64(lhs) >> amount) |
-              (static_cast<uint64_t>(Int128High64(lhs)) << (64 - amount)));
-    }
-    return lhs;
-  }
-  return MakeInt128(0,
-                    static_cast<uint64_t>(Int128High64(lhs) >> (amount - 64)));
+constexpr int128 operator<<(int128 lhs, int amount) {
+  // int64_t shifts of >= 64 are undefined, so we need some special-casing.
+  return amount >= 64
+             ? MakeInt128(
+                   static_cast<int64_t>(Int128Low64(lhs) << (amount - 64)), 0)
+         : amount == 0
+             ? lhs
+             : MakeInt128(
+                   (Int128High64(lhs) << amount) |
+                       static_cast<int64_t>(Int128Low64(lhs) >> (64 - amount)),
+                   Int128Low64(lhs) << amount);
+}
+
+constexpr int128 operator>>(int128 lhs, int amount) {
+  // int64_t shifts of >= 64 are undefined, so we need some special-casing.
+  // The (Int128High64(lhs) >> 32) >> 32 "trick" causes the the most significant
+  // int64 to be inititialized with all zeros or all ones correctly. It takes
+  // into account whether the number is negative or positive, and whether the
+  // current architecture does arithmetic or logical right shifts for negative
+  // numbers.
+  return amount >= 64
+             ? MakeInt128(
+                   (Int128High64(lhs) >> 32) >> 32,
+                   static_cast<uint64_t>(Int128High64(lhs) >> (amount - 64)))
+         : amount == 0
+             ? lhs
+             : MakeInt128(Int128High64(lhs) >> amount,
+                          (Int128Low64(lhs) >> amount) |
+                              (static_cast<uint64_t>(Int128High64(lhs))
+                               << (64 - amount)));
 }

data/third_party/abseil-cpp/absl/{base → profiling}/internal/exponential_biased.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "absl/base/internal/exponential_biased.h"
+#include "absl/profiling/internal/exponential_biased.h"
 
 #include <stdint.h>
 
@@ -26,7 +26,7 @@
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
-namespace base_internal {
+namespace profiling_internal {
 
 // The algorithm generates a random number between 0 and 1 and applies the
 // inverse cumulative distribution function for an exponential. Specifically:
@@ -64,7 +64,7 @@ int64_t ExponentialBiased::GetSkipCount(int64_t mean) {
     // Assume huge values are bias neutral, retain bias for next call.
     return std::numeric_limits<int64_t>::max() / 2;
   }
-  double value = std::round(interval);
+  double value = std::rint(interval);
   bias_ = interval - value;
   return value;
 }
@@ -88,6 +88,6 @@ void ExponentialBiased::Initialize() {
   initialized_ = true;
 }
 
-}  // namespace base_internal
+}  // namespace profiling_internal
 ABSL_NAMESPACE_END
 }  // namespace absl

data/third_party/abseil-cpp/absl/{base → profiling}/internal/exponential_biased.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_
-#define ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_
+#ifndef ABSL_PROFILING_INTERNAL_EXPONENTIAL_BIASED_H_
+#define ABSL_PROFILING_INTERNAL_EXPONENTIAL_BIASED_H_
 
 #include <stdint.h>
 
@@ -22,7 +22,7 @@
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
-namespace base_internal {
+namespace profiling_internal {
 
 // ExponentialBiased provides a small and fast random number generator for a
 // rounded exponential distribution. This generator manages very little state,
@@ -66,7 +66,7 @@ namespace base_internal {
 // Adjusting with rounding bias is relatively trivial:
 //
 //    double value = bias_ + exponential_distribution(mean)();
-//    double rounded_value = std::round(value);
+//    double rounded_value = std::rint(value);
 //    bias_ = value - rounded_value;
 //    return rounded_value;
 //
@@ -123,8 +123,8 @@ inline uint64_t ExponentialBiased::NextRandom(uint64_t rnd) {
   return (prng_mult * rnd + prng_add) & prng_mod_mask;
 }
 
-}  // namespace base_internal
+}  // namespace profiling_internal
 ABSL_NAMESPACE_END
 }  // namespace absl
 
-#endif  // ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_
+#endif  // ABSL_PROFILING_INTERNAL_EXPONENTIAL_BIASED_H_

data/third_party/abseil-cpp/absl/profiling/internal/sample_recorder.h

@@ -0,0 +1,230 @@
+// Copyright 2018 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// -----------------------------------------------------------------------------
+// File: sample_recorder.h
+// -----------------------------------------------------------------------------
+//
+// This header file defines a lock-free linked list for recording samples
+// collected from a random/stochastic process.
+//
+// This utility is internal-only. Use at your own risk.
+
+#ifndef ABSL_PROFILING_INTERNAL_SAMPLE_RECORDER_H_
+#define ABSL_PROFILING_INTERNAL_SAMPLE_RECORDER_H_
+
+#include <atomic>
+#include <cstddef>
+#include <functional>
+
+#include "absl/base/config.h"
+#include "absl/base/thread_annotations.h"
+#include "absl/synchronization/mutex.h"
+#include "absl/time/time.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace profiling_internal {
+
+// Sample<T> that has members required for linking samples in the linked list of
+// samples maintained by the SampleRecorder.  Type T defines the sampled data.
+template <typename T>
+struct Sample {
+  // Guards the ability to restore the sample to a pristine state.  This
+  // prevents races with sampling and resurrecting an object.
+  absl::Mutex init_mu;
+  T* next = nullptr;
+  T* dead ABSL_GUARDED_BY(init_mu) = nullptr;
+};
+
+// Holds samples and their associated stack traces with a soft limit of
+// `SetHashtablezMaxSamples()`.
+//
+// Thread safe.
+template <typename T>
+class SampleRecorder {
+ public:
+  SampleRecorder();
+  ~SampleRecorder();
+
+  // Registers for sampling.  Returns an opaque registration info.
+  T* Register();
+
+  // Unregisters the sample.
+  void Unregister(T* sample);
+
+  // The dispose callback will be called on all samples the moment they are
+  // being unregistered. Only affects samples that are unregistered after the
+  // callback has been set.
+  // Returns the previous callback.
+  using DisposeCallback = void (*)(const T&);
+  DisposeCallback SetDisposeCallback(DisposeCallback f);
+
+  // Iterates over all the registered `StackInfo`s.  Returning the number of
+  // samples that have been dropped.
+  int64_t Iterate(const std::function<void(const T& stack)>& f);
+
+  void SetMaxSamples(int32_t max);
+
+ private:
+  void PushNew(T* sample);
+  void PushDead(T* sample);
+  T* PopDead();
+
+  std::atomic<size_t> dropped_samples_;
+  std::atomic<size_t> size_estimate_;
+  std::atomic<int32_t> max_samples_{1 << 20};
+
+  // Intrusive lock free linked lists for tracking samples.
+  //
+  // `all_` records all samples (they are never removed from this list) and is
+  // terminated with a `nullptr`.
+  //
+  // `graveyard_.dead` is a circular linked list.  When it is empty,
+  // `graveyard_.dead == &graveyard`.  The list is circular so that
+  // every item on it (even the last) has a non-null dead pointer.  This allows
+  // `Iterate` to determine if a given sample is live or dead using only
+  // information on the sample itself.
+  //
+  // For example, nodes [A, B, C, D, E] with [A, C, E] alive and [B, D] dead
+  // looks like this (G is the Graveyard):
+  //
+  //           +---+    +---+    +---+    +---+    +---+
+  //    all -->| A |--->| B |--->| C |--->| D |--->| E |
+  //           |   |    |   |    |   |    |   |    |   |
+  //   +---+   |   | +->|   |-+  |   | +->|   |-+  |   |
+  //   | G |   +---+ |  +---+ |  +---+ |  +---+ |  +---+
+  //   |   |         |        |        |        |
+  //   |   | --------+        +--------+        |
+  //   +---+                                    |
+  //     ^                                      |
+  //     +--------------------------------------+
+  //
+  std::atomic<T*> all_;
+  T graveyard_;
+
+  std::atomic<DisposeCallback> dispose_;
+};
+
+template <typename T>
+typename SampleRecorder<T>::DisposeCallback
+SampleRecorder<T>::SetDisposeCallback(DisposeCallback f) {
+  return dispose_.exchange(f, std::memory_order_relaxed);
+}
+
+template <typename T>
+SampleRecorder<T>::SampleRecorder()
+    : dropped_samples_(0), size_estimate_(0), all_(nullptr), dispose_(nullptr) {
+  absl::MutexLock l(&graveyard_.init_mu);
+  graveyard_.dead = &graveyard_;
+}
+
+template <typename T>
+SampleRecorder<T>::~SampleRecorder() {
+  T* s = all_.load(std::memory_order_acquire);
+  while (s != nullptr) {
+    T* next = s->next;
+    delete s;
+    s = next;
+  }
+}
+
+template <typename T>
+void SampleRecorder<T>::PushNew(T* sample) {
+  sample->next = all_.load(std::memory_order_relaxed);
+  while (!all_.compare_exchange_weak(sample->next, sample,
+                                     std::memory_order_release,
+                                     std::memory_order_relaxed)) {
+  }
+}
+
+template <typename T>
+void SampleRecorder<T>::PushDead(T* sample) {
+  if (auto* dispose = dispose_.load(std::memory_order_relaxed)) {
+    dispose(*sample);
+  }
+
+  absl::MutexLock graveyard_lock(&graveyard_.init_mu);
+  absl::MutexLock sample_lock(&sample->init_mu);
+  sample->dead = graveyard_.dead;
+  graveyard_.dead = sample;
+}
+
+template <typename T>
+T* SampleRecorder<T>::PopDead() {
+  absl::MutexLock graveyard_lock(&graveyard_.init_mu);
+
+  // The list is circular, so eventually it collapses down to
+  //   graveyard_.dead == &graveyard_
+  // when it is empty.
+  T* sample = graveyard_.dead;
+  if (sample == &graveyard_) return nullptr;
+
+  absl::MutexLock sample_lock(&sample->init_mu);
+  graveyard_.dead = sample->dead;
+  sample->dead = nullptr;
+  sample->PrepareForSampling();
+  return sample;
+}
+
+template <typename T>
+T* SampleRecorder<T>::Register() {
+  int64_t size = size_estimate_.fetch_add(1, std::memory_order_relaxed);
+  if (size > max_samples_.load(std::memory_order_relaxed)) {
+    size_estimate_.fetch_sub(1, std::memory_order_relaxed);
+    dropped_samples_.fetch_add(1, std::memory_order_relaxed);
+    return nullptr;
+  }
+
+  T* sample = PopDead();
+  if (sample == nullptr) {
+    // Resurrection failed.  Hire a new warlock.
+    sample = new T();
+    PushNew(sample);
+  }
+
+  return sample;
+}
+
+template <typename T>
+void SampleRecorder<T>::Unregister(T* sample) {
+  PushDead(sample);
+  size_estimate_.fetch_sub(1, std::memory_order_relaxed);
+}
+
+template <typename T>
+int64_t SampleRecorder<T>::Iterate(
+    const std::function<void(const T& stack)>& f) {
+  T* s = all_.load(std::memory_order_acquire);
+  while (s != nullptr) {
+    absl::MutexLock l(&s->init_mu);
+    if (s->dead == nullptr) {
+      f(*s);
+    }
+    s = s->next;
+  }
+
+  return dropped_samples_.load(std::memory_order_relaxed);
+}
+
+template <typename T>
+void SampleRecorder<T>::SetMaxSamples(int32_t max) {
+  max_samples_.store(max, std::memory_order_release);
+}
+
+}  // namespace profiling_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_PROFILING_INTERNAL_SAMPLE_RECORDER_H_