grpc 1.32.0 → 1.35.0

data/third_party/abseil-cpp/absl/base/internal/invoke.h

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //
-// absl::base_internal::Invoke(f, args...) is an implementation of
+// absl::base_internal::invoke(f, args...) is an implementation of
 // INVOKE(f, args...) from section [func.require] of the C++ standard.
 //
 // [func.require]
@@ -29,7 +29,7 @@
 //    is not one of the types described in the previous item;
 // 5. f(t1, t2, ..., tN) in all other cases.
 //
-// The implementation is SFINAE-friendly: substitution failure within Invoke()
+// The implementation is SFINAE-friendly: substitution failure within invoke()
 // isn't an error.
 
 #ifndef ABSL_BASE_INTERNAL_INVOKE_H_
@@ -170,13 +170,13 @@ struct Invoker {
 
 // The result type of Invoke<F, Args...>.
 template <typename F, typename... Args>
-using InvokeT = decltype(Invoker<F, Args...>::type::Invoke(
+using invoke_result_t = decltype(Invoker<F, Args...>::type::Invoke(
     std::declval<F>(), std::declval<Args>()...));
 
 // Invoke(f, args...) is an implementation of INVOKE(f, args...) from section
 // [func.require] of the C++ standard.
 template <typename F, typename... Args>
-InvokeT<F, Args...> Invoke(F&& f, Args&&... args) {
+invoke_result_t<F, Args...> invoke(F&& f, Args&&... args) {
   return Invoker<F, Args...>::type::Invoke(std::forward<F>(f),
                                            std::forward<Args>(args)...);
 }

data/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc

@@ -598,7 +598,7 @@ static void *DoAllocWithArena(size_t request, LowLevelAlloc::Arena *arena) {
     section.Leave();
     result = &s->levels;
   }
-  ANNOTATE_MEMORY_IS_UNINITIALIZED(result, request);
+  ABSL_ANNOTATE_MEMORY_IS_UNINITIALIZED(result, request);
   return result;
 }
 

data/third_party/abseil-cpp/absl/base/internal/low_level_scheduling.h

@@ -18,6 +18,7 @@
 #ifndef ABSL_BASE_INTERNAL_LOW_LEVEL_SCHEDULING_H_
 #define ABSL_BASE_INTERNAL_LOW_LEVEL_SCHEDULING_H_
 
+#include "absl/base/internal/raw_logging.h"
 #include "absl/base/internal/scheduling_mode.h"
 #include "absl/base/macros.h"
 
@@ -29,6 +30,13 @@ extern "C" void __google_enable_rescheduling(bool disable_result);
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
+class CondVar;
+class Mutex;
+
+namespace synchronization_internal {
+int MutexDelay(int32_t c, int mode);
+}  // namespace synchronization_internal
+
 namespace base_internal {
 
 class SchedulingHelper;  // To allow use of SchedulingGuard.
@@ -76,9 +84,23 @@ class SchedulingGuard {
     bool disabled;
   };
 
-  // Access to SchedulingGuard is explicitly white-listed.
+  // A scoped helper to enable rescheduling temporarily.
+  // REQUIRES: destructor must run in same thread as constructor.
+  class ScopedEnable {
+   public:
+    ScopedEnable();
+    ~ScopedEnable();
+
+   private:
+    int scheduling_disabled_depth_;
+  };
+
+  // Access to SchedulingGuard is explicitly permitted.
+  friend class absl::CondVar;
+  friend class absl::Mutex;
   friend class SchedulingHelper;
   friend class SpinLock;
+  friend int absl::synchronization_internal::MutexDelay(int32_t c, int mode);
 
   SchedulingGuard(const SchedulingGuard&) = delete;
   SchedulingGuard& operator=(const SchedulingGuard&) = delete;
@@ -100,6 +122,12 @@ inline void SchedulingGuard::EnableRescheduling(bool /* disable_result */) {
   return;
 }
 
+inline SchedulingGuard::ScopedEnable::ScopedEnable()
+    : scheduling_disabled_depth_(0) {}
+inline SchedulingGuard::ScopedEnable::~ScopedEnable() {
+  ABSL_RAW_CHECK(scheduling_disabled_depth_ == 0, "disable unused warning");
+}
+
 }  // namespace base_internal
 ABSL_NAMESPACE_END
 }  // namespace absl

data/third_party/abseil-cpp/absl/base/internal/raw_logging.cc

@@ -69,7 +69,7 @@
 
 // TODO(gfalcon): We want raw-logging to work on as many platforms as possible.
 // Explicitly #error out when not ABSL_LOW_LEVEL_WRITE_SUPPORTED, except for a
-// whitelisted set of platforms for which we expect not to be able to raw log.
+// selected set of platforms for which we expect not to be able to raw log.
 
 ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook<
     absl::raw_logging_internal::LogPrefixHook>
@@ -227,7 +227,7 @@ bool RawLoggingFullySupported() {
 #endif  // !ABSL_LOW_LEVEL_WRITE_SUPPORTED
 }
 
-ABSL_DLL ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL
     absl::base_internal::AtomicHook<InternalLogFunction>
         internal_log_function(DefaultInternalLog);
 

data/third_party/abseil-cpp/absl/base/internal/raw_logging.h

@@ -72,10 +72,12 @@
 //
 // The API is a subset of the above: each macro only takes two arguments.  Use
 // StrCat if you need to build a richer message.
-#define ABSL_INTERNAL_LOG(severity, message)                                \
-  do {                                                                      \
-    ::absl::raw_logging_internal::internal_log_function(                    \
-        ABSL_RAW_LOGGING_INTERNAL_##severity, __FILE__, __LINE__, message); \
+#define ABSL_INTERNAL_LOG(severity, message)                             \
+  do {                                                                   \
+    constexpr const char* absl_raw_logging_internal_filename = __FILE__; \
+    ::absl::raw_logging_internal::internal_log_function(                 \
+        ABSL_RAW_LOGGING_INTERNAL_##severity,                            \
+        absl_raw_logging_internal_filename, __LINE__, message);          \
   } while (0)
 
 #define ABSL_INTERNAL_CHECK(condition, message)                    \
@@ -170,7 +172,7 @@ using InternalLogFunction = void (*)(absl::LogSeverity severity,
                                      const char* file, int line,
                                      const std::string& message);
 
-ABSL_DLL ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES extern base_internal::AtomicHook<
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL extern base_internal::AtomicHook<
     InternalLogFunction>
     internal_log_function;
 

data/third_party/abseil-cpp/absl/base/internal/spinlock.cc

@@ -66,35 +66,19 @@ void RegisterSpinLockProfiler(void (*fn)(const void *contendedlock,
   submit_profile_data.Store(fn);
 }
 
+// Static member variable definitions.
+constexpr uint32_t SpinLock::kSpinLockHeld;
+constexpr uint32_t SpinLock::kSpinLockCooperative;
+constexpr uint32_t SpinLock::kSpinLockDisabledScheduling;
+constexpr uint32_t SpinLock::kSpinLockSleeper;
+constexpr uint32_t SpinLock::kWaitTimeMask;
+
 // Uncommon constructors.
 SpinLock::SpinLock(base_internal::SchedulingMode mode)
     : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {
   ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
 }
 
-SpinLock::SpinLock(base_internal::LinkerInitialized,
-                   base_internal::SchedulingMode mode) {
-  ABSL_TSAN_MUTEX_CREATE(this, 0);
-  if (IsCooperative(mode)) {
-    InitLinkerInitializedAndCooperative();
-  }
-  // Otherwise, lockword_ is already initialized.
-}
-
-// Static (linker initialized) spinlocks always start life as functional
-// non-cooperative locks.  When their static constructor does run, it will call
-// this initializer to augment the lockword with the cooperative bit.  By
-// actually taking the lock when we do this we avoid the need for an atomic
-// operation in the regular unlock path.
-//
-// SlowLock() must be careful to re-test for this bit so that any outstanding
-// waiters may be upgraded to cooperative status.
-void SpinLock::InitLinkerInitializedAndCooperative() {
-  Lock();
-  lockword_.fetch_or(kSpinLockCooperative, std::memory_order_relaxed);
-  Unlock();
-}
-
 // Monitor the lock to see if its value changes within some time period
 // (adaptive_spin_count loop iterations). The last value read from the lock
 // is returned from the method.
@@ -121,6 +105,14 @@ void SpinLock::SlowLock() {
   if ((lock_value & kSpinLockHeld) == 0) {
     return;
   }
+
+  base_internal::SchedulingMode scheduling_mode;
+  if ((lock_value & kSpinLockCooperative) != 0) {
+    scheduling_mode = base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL;
+  } else {
+    scheduling_mode = base_internal::SCHEDULE_KERNEL_ONLY;
+  }
+
   // The lock was not obtained initially, so this thread needs to wait for
   // it.  Record the current timestamp in the local variable wait_start_time
   // so the total wait time can be stored in the lockword once this thread
@@ -151,12 +143,6 @@ void SpinLock::SlowLock() {
       }
     }
 
-    base_internal::SchedulingMode scheduling_mode;
-    if ((lock_value & kSpinLockCooperative) != 0) {
-      scheduling_mode = base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL;
-    } else {
-      scheduling_mode = base_internal::SCHEDULE_KERNEL_ONLY;
-    }
     // SpinLockDelay() calls into fiber scheduler, we need to see
     // synchronization there to avoid false positives.
     ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0);
@@ -190,30 +176,32 @@ void SpinLock::SlowUnlock(uint32_t lock_value) {
 // We use the upper 29 bits of the lock word to store the time spent waiting to
 // acquire this lock.  This is reported by contentionz profiling.  Since the
 // lower bits of the cycle counter wrap very quickly on high-frequency
-// processors we divide to reduce the granularity to 2^PROFILE_TIMESTAMP_SHIFT
+// processors we divide to reduce the granularity to 2^kProfileTimestampShift
 // sized units.  On a 4Ghz machine this will lose track of wait times greater
 // than (2^29/4 Ghz)*128 =~ 17.2 seconds.  Such waits should be extremely rare.
-enum { PROFILE_TIMESTAMP_SHIFT = 7 };
-enum { LOCKWORD_RESERVED_SHIFT = 3 };  // We currently reserve the lower 3 bits.
+static constexpr int kProfileTimestampShift = 7;
+
+// We currently reserve the lower 3 bits.
+static constexpr int kLockwordReservedShift = 3;
 
 uint32_t SpinLock::EncodeWaitCycles(int64_t wait_start_time,
                                     int64_t wait_end_time) {
   static const int64_t kMaxWaitTime =
-      std::numeric_limits<uint32_t>::max() >> LOCKWORD_RESERVED_SHIFT;
+      std::numeric_limits<uint32_t>::max() >> kLockwordReservedShift;
   int64_t scaled_wait_time =
-      (wait_end_time - wait_start_time) >> PROFILE_TIMESTAMP_SHIFT;
+      (wait_end_time - wait_start_time) >> kProfileTimestampShift;
 
   // Return a representation of the time spent waiting that can be stored in
   // the lock word's upper bits.
   uint32_t clamped = static_cast<uint32_t>(
-      std::min(scaled_wait_time, kMaxWaitTime) << LOCKWORD_RESERVED_SHIFT);
+      std::min(scaled_wait_time, kMaxWaitTime) << kLockwordReservedShift);
 
   if (clamped == 0) {
     return kSpinLockSleeper;  // Just wake waiters, but don't record contention.
   }
   // Bump up value if necessary to avoid returning kSpinLockSleeper.
   const uint32_t kMinWaitTime =
-      kSpinLockSleeper + (1 << LOCKWORD_RESERVED_SHIFT);
+      kSpinLockSleeper + (1 << kLockwordReservedShift);
   if (clamped == kSpinLockSleeper) {
     return kMinWaitTime;
   }
@@ -224,8 +212,7 @@ uint64_t SpinLock::DecodeWaitCycles(uint32_t lock_value) {
   // Cast to uint32_t first to ensure bits [63:32] are cleared.
   const uint64_t scaled_wait_time =
       static_cast<uint32_t>(lock_value & kWaitTimeMask);
-  return scaled_wait_time
-      << (PROFILE_TIMESTAMP_SHIFT - LOCKWORD_RESERVED_SHIFT);
+  return scaled_wait_time << (kProfileTimestampShift - kLockwordReservedShift);
 }
 
 }  // namespace base_internal

data/third_party/abseil-cpp/absl/base/internal/spinlock.h

@@ -36,6 +36,7 @@
 #include <atomic>
 
 #include "absl/base/attributes.h"
+#include "absl/base/const_init.h"
 #include "absl/base/dynamic_annotations.h"
 #include "absl/base/internal/low_level_scheduling.h"
 #include "absl/base/internal/raw_logging.h"
@@ -55,29 +56,22 @@ class ABSL_LOCKABLE SpinLock {
     ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
   }
 
-  // Special constructor for use with static SpinLock objects.  E.g.,
-  //
-  //    static SpinLock lock(base_internal::kLinkerInitialized);
-  //
-  // When initialized using this constructor, we depend on the fact
-  // that the linker has already initialized the memory appropriately. The lock
-  // is initialized in non-cooperative mode.
-  //
-  // A SpinLock constructed like this can be freely used from global
-  // initializers without worrying about the order in which global
-  // initializers run.
-  explicit SpinLock(base_internal::LinkerInitialized) {
-    // Does nothing; lockword_ is already initialized
-    ABSL_TSAN_MUTEX_CREATE(this, 0);
-  }
-
   // Constructors that allow non-cooperative spinlocks to be created for use
   // inside thread schedulers.  Normal clients should not use these.
   explicit SpinLock(base_internal::SchedulingMode mode);
-  SpinLock(base_internal::LinkerInitialized,
-           base_internal::SchedulingMode mode);
 
+  // Constructor for global SpinLock instances.  See absl/base/const_init.h.
+  constexpr SpinLock(absl::ConstInitType, base_internal::SchedulingMode mode)
+      : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {}
+
+  // For global SpinLock instances prefer trivial destructor when possible.
+  // Default but non-trivial destructor in some build configurations causes an
+  // extra static initializer.
+#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE
   ~SpinLock() { ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); }
+#else
+  ~SpinLock() = default;
+#endif
 
   // Acquire this SpinLock.
   inline void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() {
@@ -148,12 +142,13 @@ class ABSL_LOCKABLE SpinLock {
   // bit[1] encodes whether a lock uses cooperative scheduling.
   // bit[2] encodes whether a lock disables scheduling.
   // bit[3:31] encodes time a lock spent on waiting as a 29-bit unsigned int.
-  enum { kSpinLockHeld = 1 };
-  enum { kSpinLockCooperative = 2 };
-  enum { kSpinLockDisabledScheduling = 4 };
-  enum { kSpinLockSleeper = 8 };
-  enum { kWaitTimeMask =                      // Includes kSpinLockSleeper.
-    ~(kSpinLockHeld | kSpinLockCooperative | kSpinLockDisabledScheduling) };
+  static constexpr uint32_t kSpinLockHeld = 1;
+  static constexpr uint32_t kSpinLockCooperative = 2;
+  static constexpr uint32_t kSpinLockDisabledScheduling = 4;
+  static constexpr uint32_t kSpinLockSleeper = 8;
+  // Includes kSpinLockSleeper.
+  static constexpr uint32_t kWaitTimeMask =
+      ~(kSpinLockHeld | kSpinLockCooperative | kSpinLockDisabledScheduling);
 
   // Returns true if the provided scheduling mode is cooperative.
   static constexpr bool IsCooperative(
@@ -162,7 +157,6 @@ class ABSL_LOCKABLE SpinLock {
   }
 
   uint32_t TryLockInternal(uint32_t lock_value, uint32_t wait_cycles);
-  void InitLinkerInitializedAndCooperative();
   void SlowLock() ABSL_ATTRIBUTE_COLD;
   void SlowUnlock(uint32_t lock_value) ABSL_ATTRIBUTE_COLD;
   uint32_t SpinLoop();

data/third_party/abseil-cpp/absl/base/internal/spinlock_linux.inc

@@ -46,6 +46,14 @@ static_assert(sizeof(std::atomic<uint32_t>) == sizeof(int),
 #endif
 #endif
 
+#if defined(__NR_futex_time64) && !defined(SYS_futex_time64)
+#define SYS_futex_time64 __NR_futex_time64
+#endif
+
+#if defined(SYS_futex_time64) && !defined(SYS_futex)
+#define SYS_futex SYS_futex_time64
+#endif
+
 extern "C" {
 
 ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay(

data/third_party/abseil-cpp/absl/base/internal/sysinfo.cc

@@ -39,6 +39,7 @@
 #endif
 
 #include <string.h>
+
 #include <cassert>
 #include <cstdint>
 #include <cstdio>
@@ -50,9 +51,11 @@
 #include <vector>
 
 #include "absl/base/call_once.h"
+#include "absl/base/config.h"
 #include "absl/base/internal/raw_logging.h"
 #include "absl/base/internal/spinlock.h"
 #include "absl/base/internal/unscaledcycleclock.h"
+#include "absl/base/thread_annotations.h"
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
@@ -72,6 +75,12 @@ static int GetNumCPUs() {
 #if defined(_WIN32)
 
 static double GetNominalCPUFrequency() {
+#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) && \
+    !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
+  // UWP apps don't have access to the registry and currently don't provide an
+  // API informing about CPU nominal frequency.
+  return 1.0;
+#else
 #pragma comment(lib, "advapi32.lib")  // For Reg* functions.
   HKEY key;
   // Use the Reg* functions rather than the SH functions because shlwapi.dll
@@ -91,6 +100,7 @@ static double GetNominalCPUFrequency() {
     }
   }
   return 1.0;
+#endif  // WINAPI_PARTITION_APP && !WINAPI_PARTITION_DESKTOP
 }
 
 #elif defined(CTL_HW) && defined(HW_CPU_FREQ)
@@ -336,15 +346,16 @@ pid_t GetTID() {
 #else
 
 // Fallback implementation of GetTID using pthread_getspecific.
-static once_flag tid_once;
-static pthread_key_t tid_key;
-static absl::base_internal::SpinLock tid_lock(
-    absl::base_internal::kLinkerInitialized);
+ABSL_CONST_INIT static once_flag tid_once;
+ABSL_CONST_INIT static pthread_key_t tid_key;
+ABSL_CONST_INIT static absl::base_internal::SpinLock tid_lock(
+    absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
 
 // We set a bit per thread in this array to indicate that an ID is in
 // use. ID 0 is unused because it is the default value returned by
 // pthread_getspecific().
-static std::vector<uint32_t>* tid_array GUARDED_BY(tid_lock) = nullptr;
+ABSL_CONST_INIT static std::vector<uint32_t> *tid_array
+    ABSL_GUARDED_BY(tid_lock) = nullptr;
 static constexpr int kBitsPerWord = 32;  // tid_array is uint32_t.
 
 // Returns the TID to tid_array.
@@ -411,6 +422,18 @@ pid_t GetTID() {
 
 #endif
 
+// GetCachedTID() caches the thread ID in thread-local storage (which is a
+// userspace construct) to avoid unnecessary system calls. Without this caching,
+// it can take roughly 98ns, while it takes roughly 1ns with this caching.
+pid_t GetCachedTID() {
+#if ABSL_HAVE_THREAD_LOCAL
+  static thread_local pid_t thread_id = GetTID();
+  return thread_id;
+#else
+  return GetTID();
+#endif  // ABSL_HAVE_THREAD_LOCAL
+}
+
 }  // namespace base_internal
 ABSL_NAMESPACE_END
 }  // namespace absl

data/third_party/abseil-cpp/absl/base/internal/sysinfo.h

@@ -30,6 +30,7 @@
 
 #include <cstdint>
 
+#include "absl/base/config.h"
 #include "absl/base/port.h"
 
 namespace absl {
@@ -59,6 +60,13 @@ using pid_t = uint32_t;
 #endif
 pid_t GetTID();
 
+// Like GetTID(), but caches the result in thread-local storage in order
+// to avoid unnecessary system calls. Note that there are some cases where
+// one must call through to GetTID directly, which is why this exists as a
+// separate function. For example, GetCachedTID() is not safe to call in
+// an asynchronous signal-handling context nor right after a call to fork().
+pid_t GetCachedTID();
+
 }  // namespace base_internal
 ABSL_NAMESPACE_END
 }  // namespace absl