grpc 1.28.0 → 1.36.0

data/third_party/abseil-cpp/absl/base/internal/low_level_alloc.h

@@ -0,0 +1,126 @@
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef ABSL_BASE_INTERNAL_LOW_LEVEL_ALLOC_H_
+#define ABSL_BASE_INTERNAL_LOW_LEVEL_ALLOC_H_
+
+// A simple thread-safe memory allocator that does not depend on
+// mutexes or thread-specific data.  It is intended to be used
+// sparingly, and only when malloc() would introduce an unwanted
+// dependency, such as inside the heap-checker, or the Mutex
+// implementation.
+
+// IWYU pragma: private, include "base/low_level_alloc.h"
+
+#include <sys/types.h>
+
+#include <cstdint>
+
+#include "absl/base/attributes.h"
+#include "absl/base/config.h"
+
+// LowLevelAlloc requires that the platform support low-level
+// allocation of virtual memory. Platforms lacking this cannot use
+// LowLevelAlloc.
+#ifdef ABSL_LOW_LEVEL_ALLOC_MISSING
+#error ABSL_LOW_LEVEL_ALLOC_MISSING cannot be directly set
+#elif !defined(ABSL_HAVE_MMAP) && !defined(_WIN32)
+#define ABSL_LOW_LEVEL_ALLOC_MISSING 1
+#endif
+
+// Using LowLevelAlloc with kAsyncSignalSafe isn't supported on Windows or
+// asm.js / WebAssembly.
+// See https://kripken.github.io/emscripten-site/docs/porting/pthreads.html
+// for more information.
+#ifdef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
+#error ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING cannot be directly set
+#elif defined(_WIN32) || defined(__asmjs__) || defined(__wasm__)
+#define ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING 1
+#endif
+
+#include <cstddef>
+
+#include "absl/base/port.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace base_internal {
+
+class LowLevelAlloc {
+ public:
+  struct Arena;       // an arena from which memory may be allocated
+
+  // Returns a pointer to a block of at least "request" bytes
+  // that have been newly allocated from the specific arena.
+  // for Alloc() call the DefaultArena() is used.
+  // Returns 0 if passed request==0.
+  // Does not return 0 under other circumstances; it crashes if memory
+  // is not available.
+  static void *Alloc(size_t request) ABSL_ATTRIBUTE_SECTION(malloc_hook);
+  static void *AllocWithArena(size_t request, Arena *arena)
+      ABSL_ATTRIBUTE_SECTION(malloc_hook);
+
+  // Deallocates a region of memory that was previously allocated with
+  // Alloc().   Does nothing if passed 0.   "s" must be either 0,
+  // or must have been returned from a call to Alloc() and not yet passed to
+  // Free() since that call to Alloc().  The space is returned to the arena
+  // from which it was allocated.
+  static void Free(void *s) ABSL_ATTRIBUTE_SECTION(malloc_hook);
+
+  // ABSL_ATTRIBUTE_SECTION(malloc_hook) for Alloc* and Free
+  // are to put all callers of MallocHook::Invoke* in this module
+  // into special section,
+  // so that MallocHook::GetCallerStackTrace can function accurately.
+
+  // Create a new arena.
+  // The root metadata for the new arena is allocated in the
+  // meta_data_arena; the DefaultArena() can be passed for meta_data_arena.
+  // These values may be ored into flags:
+  enum {
+    // Report calls to Alloc() and Free() via the MallocHook interface.
+    // Set in the DefaultArena.
+    kCallMallocHook = 0x0001,
+
+#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
+    // Make calls to Alloc(), Free() be async-signal-safe. Not set in
+    // DefaultArena(). Not supported on all platforms.
+    kAsyncSignalSafe = 0x0002,
+#endif
+  };
+  // Construct a new arena.  The allocation of the underlying metadata honors
+  // the provided flags.  For example, the call NewArena(kAsyncSignalSafe)
+  // is itself async-signal-safe, as well as generatating an arena that provides
+  // async-signal-safe Alloc/Free.
+  static Arena *NewArena(int32_t flags);
+
+  // Destroys an arena allocated by NewArena and returns true,
+  // provided no allocated blocks remain in the arena.
+  // If allocated blocks remain in the arena, does nothing and
+  // returns false.
+  // It is illegal to attempt to destroy the DefaultArena().
+  static bool DeleteArena(Arena *arena);
+
+  // The default arena that always exists.
+  static Arena *DefaultArena();
+
+ private:
+  LowLevelAlloc();      // no instances
+};
+
+}  // namespace base_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_BASE_INTERNAL_LOW_LEVEL_ALLOC_H_

data/third_party/abseil-cpp/absl/base/internal/low_level_scheduling.h

@@ -18,6 +18,7 @@
 #ifndef ABSL_BASE_INTERNAL_LOW_LEVEL_SCHEDULING_H_
 #define ABSL_BASE_INTERNAL_LOW_LEVEL_SCHEDULING_H_
 
+#include "absl/base/internal/raw_logging.h"
 #include "absl/base/internal/scheduling_mode.h"
 #include "absl/base/macros.h"
 
@@ -29,6 +30,13 @@ extern "C" void __google_enable_rescheduling(bool disable_result);
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
+class CondVar;
+class Mutex;
+
+namespace synchronization_internal {
+int MutexDelay(int32_t c, int mode);
+}  // namespace synchronization_internal
+
 namespace base_internal {
 
 class SchedulingHelper;  // To allow use of SchedulingGuard.
@@ -76,9 +84,23 @@ class SchedulingGuard {
     bool disabled;
   };
 
-  // Access to SchedulingGuard is explicitly white-listed.
+  // A scoped helper to enable rescheduling temporarily.
+  // REQUIRES: destructor must run in same thread as constructor.
+  class ScopedEnable {
+   public:
+    ScopedEnable();
+    ~ScopedEnable();
+
+   private:
+    int scheduling_disabled_depth_;
+  };
+
+  // Access to SchedulingGuard is explicitly permitted.
+  friend class absl::CondVar;
+  friend class absl::Mutex;
   friend class SchedulingHelper;
   friend class SpinLock;
+  friend int absl::synchronization_internal::MutexDelay(int32_t c, int mode);
 
   SchedulingGuard(const SchedulingGuard&) = delete;
   SchedulingGuard& operator=(const SchedulingGuard&) = delete;
@@ -100,6 +122,12 @@ inline void SchedulingGuard::EnableRescheduling(bool /* disable_result */) {
   return;
 }
 
+inline SchedulingGuard::ScopedEnable::ScopedEnable()
+    : scheduling_disabled_depth_(0) {}
+inline SchedulingGuard::ScopedEnable::~ScopedEnable() {
+  ABSL_RAW_CHECK(scheduling_disabled_depth_ == 0, "disable unused warning");
+}
+
 }  // namespace base_internal
 ABSL_NAMESPACE_END
 }  // namespace absl

data/third_party/abseil-cpp/absl/base/internal/raw_logging.cc

@@ -69,7 +69,7 @@
 
 // TODO(gfalcon): We want raw-logging to work on as many platforms as possible.
 // Explicitly #error out when not ABSL_LOW_LEVEL_WRITE_SUPPORTED, except for a
-// whitelisted set of platforms for which we expect not to be able to raw log.
+// selected set of platforms for which we expect not to be able to raw log.
 
 ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook<
     absl::raw_logging_internal::LogPrefixHook>
@@ -227,7 +227,7 @@ bool RawLoggingFullySupported() {
 #endif  // !ABSL_LOW_LEVEL_WRITE_SUPPORTED
 }
 
-ABSL_DLL ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL
     absl::base_internal::AtomicHook<InternalLogFunction>
         internal_log_function(DefaultInternalLog);
 

data/third_party/abseil-cpp/absl/base/internal/raw_logging.h

@@ -72,10 +72,12 @@
 //
 // The API is a subset of the above: each macro only takes two arguments.  Use
 // StrCat if you need to build a richer message.
-#define ABSL_INTERNAL_LOG(severity, message)                                \
-  do {                                                                      \
-    ::absl::raw_logging_internal::internal_log_function(                    \
-        ABSL_RAW_LOGGING_INTERNAL_##severity, __FILE__, __LINE__, message); \
+#define ABSL_INTERNAL_LOG(severity, message)                             \
+  do {                                                                   \
+    constexpr const char* absl_raw_logging_internal_filename = __FILE__; \
+    ::absl::raw_logging_internal::internal_log_function(                 \
+        ABSL_RAW_LOGGING_INTERNAL_##severity,                            \
+        absl_raw_logging_internal_filename, __LINE__, message);          \
   } while (0)
 
 #define ABSL_INTERNAL_CHECK(condition, message)                    \
@@ -170,7 +172,7 @@ using InternalLogFunction = void (*)(absl::LogSeverity severity,
                                      const char* file, int line,
                                      const std::string& message);
 
-ABSL_DLL ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES extern base_internal::AtomicHook<
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL extern base_internal::AtomicHook<
     InternalLogFunction>
     internal_log_function;
 

data/third_party/abseil-cpp/absl/base/internal/spinlock.cc

@@ -66,35 +66,19 @@ void RegisterSpinLockProfiler(void (*fn)(const void *contendedlock,
   submit_profile_data.Store(fn);
 }
 
+// Static member variable definitions.
+constexpr uint32_t SpinLock::kSpinLockHeld;
+constexpr uint32_t SpinLock::kSpinLockCooperative;
+constexpr uint32_t SpinLock::kSpinLockDisabledScheduling;
+constexpr uint32_t SpinLock::kSpinLockSleeper;
+constexpr uint32_t SpinLock::kWaitTimeMask;
+
 // Uncommon constructors.
 SpinLock::SpinLock(base_internal::SchedulingMode mode)
     : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {
   ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
 }
 
-SpinLock::SpinLock(base_internal::LinkerInitialized,
-                   base_internal::SchedulingMode mode) {
-  ABSL_TSAN_MUTEX_CREATE(this, 0);
-  if (IsCooperative(mode)) {
-    InitLinkerInitializedAndCooperative();
-  }
-  // Otherwise, lockword_ is already initialized.
-}
-
-// Static (linker initialized) spinlocks always start life as functional
-// non-cooperative locks.  When their static constructor does run, it will call
-// this initializer to augment the lockword with the cooperative bit.  By
-// actually taking the lock when we do this we avoid the need for an atomic
-// operation in the regular unlock path.
-//
-// SlowLock() must be careful to re-test for this bit so that any outstanding
-// waiters may be upgraded to cooperative status.
-void SpinLock::InitLinkerInitializedAndCooperative() {
-  Lock();
-  lockword_.fetch_or(kSpinLockCooperative, std::memory_order_relaxed);
-  Unlock();
-}
-
 // Monitor the lock to see if its value changes within some time period
 // (adaptive_spin_count loop iterations). The last value read from the lock
 // is returned from the method.
@@ -121,6 +105,14 @@ void SpinLock::SlowLock() {
   if ((lock_value & kSpinLockHeld) == 0) {
     return;
   }
+
+  base_internal::SchedulingMode scheduling_mode;
+  if ((lock_value & kSpinLockCooperative) != 0) {
+    scheduling_mode = base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL;
+  } else {
+    scheduling_mode = base_internal::SCHEDULE_KERNEL_ONLY;
+  }
+
   // The lock was not obtained initially, so this thread needs to wait for
   // it.  Record the current timestamp in the local variable wait_start_time
   // so the total wait time can be stored in the lockword once this thread
@@ -151,12 +143,6 @@ void SpinLock::SlowLock() {
       }
     }
 
-    base_internal::SchedulingMode scheduling_mode;
-    if ((lock_value & kSpinLockCooperative) != 0) {
-      scheduling_mode = base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL;
-    } else {
-      scheduling_mode = base_internal::SCHEDULE_KERNEL_ONLY;
-    }
     // SpinLockDelay() calls into fiber scheduler, we need to see
     // synchronization there to avoid false positives.
     ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0);
@@ -190,30 +176,32 @@ void SpinLock::SlowUnlock(uint32_t lock_value) {
 // We use the upper 29 bits of the lock word to store the time spent waiting to
 // acquire this lock.  This is reported by contentionz profiling.  Since the
 // lower bits of the cycle counter wrap very quickly on high-frequency
-// processors we divide to reduce the granularity to 2^PROFILE_TIMESTAMP_SHIFT
+// processors we divide to reduce the granularity to 2^kProfileTimestampShift
 // sized units.  On a 4Ghz machine this will lose track of wait times greater
 // than (2^29/4 Ghz)*128 =~ 17.2 seconds.  Such waits should be extremely rare.
-enum { PROFILE_TIMESTAMP_SHIFT = 7 };
-enum { LOCKWORD_RESERVED_SHIFT = 3 };  // We currently reserve the lower 3 bits.
+static constexpr int kProfileTimestampShift = 7;
+
+// We currently reserve the lower 3 bits.
+static constexpr int kLockwordReservedShift = 3;
 
 uint32_t SpinLock::EncodeWaitCycles(int64_t wait_start_time,
                                     int64_t wait_end_time) {
   static const int64_t kMaxWaitTime =
-      std::numeric_limits<uint32_t>::max() >> LOCKWORD_RESERVED_SHIFT;
+      std::numeric_limits<uint32_t>::max() >> kLockwordReservedShift;
   int64_t scaled_wait_time =
-      (wait_end_time - wait_start_time) >> PROFILE_TIMESTAMP_SHIFT;
+      (wait_end_time - wait_start_time) >> kProfileTimestampShift;
 
   // Return a representation of the time spent waiting that can be stored in
   // the lock word's upper bits.
   uint32_t clamped = static_cast<uint32_t>(
-      std::min(scaled_wait_time, kMaxWaitTime) << LOCKWORD_RESERVED_SHIFT);
+      std::min(scaled_wait_time, kMaxWaitTime) << kLockwordReservedShift);
 
   if (clamped == 0) {
     return kSpinLockSleeper;  // Just wake waiters, but don't record contention.
   }
   // Bump up value if necessary to avoid returning kSpinLockSleeper.
   const uint32_t kMinWaitTime =
-      kSpinLockSleeper + (1 << LOCKWORD_RESERVED_SHIFT);
+      kSpinLockSleeper + (1 << kLockwordReservedShift);
   if (clamped == kSpinLockSleeper) {
     return kMinWaitTime;
   }
@@ -224,8 +212,7 @@ uint64_t SpinLock::DecodeWaitCycles(uint32_t lock_value) {
   // Cast to uint32_t first to ensure bits [63:32] are cleared.
   const uint64_t scaled_wait_time =
       static_cast<uint32_t>(lock_value & kWaitTimeMask);
-  return scaled_wait_time
-      << (PROFILE_TIMESTAMP_SHIFT - LOCKWORD_RESERVED_SHIFT);
+  return scaled_wait_time << (kProfileTimestampShift - kLockwordReservedShift);
 }
 
 }  // namespace base_internal

data/third_party/abseil-cpp/absl/base/internal/spinlock.h

@@ -36,6 +36,7 @@
 #include <atomic>
 
 #include "absl/base/attributes.h"
+#include "absl/base/const_init.h"
 #include "absl/base/dynamic_annotations.h"
 #include "absl/base/internal/low_level_scheduling.h"
 #include "absl/base/internal/raw_logging.h"
@@ -55,29 +56,22 @@ class ABSL_LOCKABLE SpinLock {
     ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
   }
 
-  // Special constructor for use with static SpinLock objects.  E.g.,
-  //
-  //    static SpinLock lock(base_internal::kLinkerInitialized);
-  //
-  // When initialized using this constructor, we depend on the fact
-  // that the linker has already initialized the memory appropriately. The lock
-  // is initialized in non-cooperative mode.
-  //
-  // A SpinLock constructed like this can be freely used from global
-  // initializers without worrying about the order in which global
-  // initializers run.
-  explicit SpinLock(base_internal::LinkerInitialized) {
-    // Does nothing; lockword_ is already initialized
-    ABSL_TSAN_MUTEX_CREATE(this, 0);
-  }
-
   // Constructors that allow non-cooperative spinlocks to be created for use
   // inside thread schedulers.  Normal clients should not use these.
   explicit SpinLock(base_internal::SchedulingMode mode);
-  SpinLock(base_internal::LinkerInitialized,
-           base_internal::SchedulingMode mode);
 
+  // Constructor for global SpinLock instances.  See absl/base/const_init.h.
+  constexpr SpinLock(absl::ConstInitType, base_internal::SchedulingMode mode)
+      : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {}
+
+  // For global SpinLock instances prefer trivial destructor when possible.
+  // Default but non-trivial destructor in some build configurations causes an
+  // extra static initializer.
+#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE
   ~SpinLock() { ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); }
+#else
+  ~SpinLock() = default;
+#endif
 
   // Acquire this SpinLock.
   inline void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() {
@@ -148,12 +142,13 @@ class ABSL_LOCKABLE SpinLock {
   // bit[1] encodes whether a lock uses cooperative scheduling.
   // bit[2] encodes whether a lock disables scheduling.
   // bit[3:31] encodes time a lock spent on waiting as a 29-bit unsigned int.
-  enum { kSpinLockHeld = 1 };
-  enum { kSpinLockCooperative = 2 };
-  enum { kSpinLockDisabledScheduling = 4 };
-  enum { kSpinLockSleeper = 8 };
-  enum { kWaitTimeMask =                      // Includes kSpinLockSleeper.
-    ~(kSpinLockHeld | kSpinLockCooperative | kSpinLockDisabledScheduling) };
+  static constexpr uint32_t kSpinLockHeld = 1;
+  static constexpr uint32_t kSpinLockCooperative = 2;
+  static constexpr uint32_t kSpinLockDisabledScheduling = 4;
+  static constexpr uint32_t kSpinLockSleeper = 8;
+  // Includes kSpinLockSleeper.
+  static constexpr uint32_t kWaitTimeMask =
+      ~(kSpinLockHeld | kSpinLockCooperative | kSpinLockDisabledScheduling);
 
   // Returns true if the provided scheduling mode is cooperative.
   static constexpr bool IsCooperative(
@@ -162,7 +157,6 @@ class ABSL_LOCKABLE SpinLock {
   }
 
   uint32_t TryLockInternal(uint32_t lock_value, uint32_t wait_cycles);
-  void InitLinkerInitializedAndCooperative();
   void SlowLock() ABSL_ATTRIBUTE_COLD;
   void SlowUnlock(uint32_t lock_value) ABSL_ATTRIBUTE_COLD;
   uint32_t SpinLoop();

data/third_party/abseil-cpp/absl/base/internal/spinlock_linux.inc

@@ -46,6 +46,14 @@ static_assert(sizeof(std::atomic<uint32_t>) == sizeof(int),
 #endif
 #endif
 
+#if defined(__NR_futex_time64) && !defined(SYS_futex_time64)
+#define SYS_futex_time64 __NR_futex_time64
+#endif
+
+#if defined(SYS_futex_time64) && !defined(SYS_futex)
+#define SYS_futex SYS_futex_time64
+#endif
+
 extern "C" {
 
 ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay(

data/third_party/abseil-cpp/absl/base/internal/sysinfo.cc

@@ -39,6 +39,7 @@
 #endif
 
 #include <string.h>
+
 #include <cassert>
 #include <cstdint>
 #include <cstdio>
@@ -50,9 +51,11 @@
 #include <vector>
 
 #include "absl/base/call_once.h"
+#include "absl/base/config.h"
 #include "absl/base/internal/raw_logging.h"
 #include "absl/base/internal/spinlock.h"
 #include "absl/base/internal/unscaledcycleclock.h"
+#include "absl/base/thread_annotations.h"
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
@@ -72,6 +75,12 @@ static int GetNumCPUs() {
 #if defined(_WIN32)
 
 static double GetNominalCPUFrequency() {
+#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) && \
+    !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
+  // UWP apps don't have access to the registry and currently don't provide an
+  // API informing about CPU nominal frequency.
+  return 1.0;
+#else
 #pragma comment(lib, "advapi32.lib")  // For Reg* functions.
   HKEY key;
   // Use the Reg* functions rather than the SH functions because shlwapi.dll
@@ -91,6 +100,7 @@ static double GetNominalCPUFrequency() {
     }
   }
   return 1.0;
+#endif  // WINAPI_PARTITION_APP && !WINAPI_PARTITION_DESKTOP
 }
 
 #elif defined(CTL_HW) && defined(HW_CPU_FREQ)
@@ -336,15 +346,16 @@ pid_t GetTID() {
 #else
 
 // Fallback implementation of GetTID using pthread_getspecific.
-static once_flag tid_once;
-static pthread_key_t tid_key;
-static absl::base_internal::SpinLock tid_lock(
-    absl::base_internal::kLinkerInitialized);
+ABSL_CONST_INIT static once_flag tid_once;
+ABSL_CONST_INIT static pthread_key_t tid_key;
+ABSL_CONST_INIT static absl::base_internal::SpinLock tid_lock(
+    absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
 
 // We set a bit per thread in this array to indicate that an ID is in
 // use. ID 0 is unused because it is the default value returned by
 // pthread_getspecific().
-static std::vector<uint32_t>* tid_array GUARDED_BY(tid_lock) = nullptr;
+ABSL_CONST_INIT static std::vector<uint32_t> *tid_array
+    ABSL_GUARDED_BY(tid_lock) = nullptr;
 static constexpr int kBitsPerWord = 32;  // tid_array is uint32_t.
 
 // Returns the TID to tid_array.
@@ -411,6 +422,18 @@ pid_t GetTID() {
 
 #endif
 
+// GetCachedTID() caches the thread ID in thread-local storage (which is a
+// userspace construct) to avoid unnecessary system calls. Without this caching,
+// it can take roughly 98ns, while it takes roughly 1ns with this caching.
+pid_t GetCachedTID() {
+#if ABSL_HAVE_THREAD_LOCAL
+  static thread_local pid_t thread_id = GetTID();
+  return thread_id;
+#else
+  return GetTID();
+#endif  // ABSL_HAVE_THREAD_LOCAL
+}
+
 }  // namespace base_internal
 ABSL_NAMESPACE_END
 }  // namespace absl