grpc 1.37.1 → 1.38.0.pre1

data/third_party/abseil-cpp/absl/base/internal/raw_logging.cc

@@ -67,28 +67,32 @@
 #undef ABSL_HAVE_RAW_IO
 #endif
 
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace raw_logging_internal {
+namespace {
+
 // TODO(gfalcon): We want raw-logging to work on as many platforms as possible.
-// Explicitly #error out when not ABSL_LOW_LEVEL_WRITE_SUPPORTED, except for a
-// selected set of platforms for which we expect not to be able to raw log.
+// Explicitly `#error` out when not `ABSL_LOW_LEVEL_WRITE_SUPPORTED`, except for
+// a selected set of platforms for which we expect not to be able to raw log.
 
-ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook<
-    absl::raw_logging_internal::LogPrefixHook>
-    log_prefix_hook;
-ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook<
-    absl::raw_logging_internal::AbortHook>
-    abort_hook;
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES
+    absl::base_internal::AtomicHook<LogPrefixHook>
+        log_prefix_hook;
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES
+    absl::base_internal::AtomicHook<AbortHook>
+        abort_hook;
 
 #ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED
-static const char kTruncated[] = " ... (message truncated)\n";
+constexpr char kTruncated[] = " ... (message truncated)\n";
 
 // sprintf the format to the buffer, adjusting *buf and *size to reflect the
 // consumed bytes, and return whether the message fit without truncation.  If
 // truncation occurred, if possible leave room in the buffer for the message
 // kTruncated[].
-inline static bool VADoRawLog(char** buf, int* size, const char* format,
-                              va_list ap) ABSL_PRINTF_ATTRIBUTE(3, 0);
-inline static bool VADoRawLog(char** buf, int* size,
-                              const char* format, va_list ap) {
+bool VADoRawLog(char** buf, int* size, const char* format, va_list ap)
+    ABSL_PRINTF_ATTRIBUTE(3, 0);
+bool VADoRawLog(char** buf, int* size, const char* format, va_list ap) {
   int n = vsnprintf(*buf, *size, format, ap);
   bool result = true;
   if (n < 0 || n > *size) {
@@ -96,7 +100,7 @@ inline static bool VADoRawLog(char** buf, int* size,
     if (static_cast<size_t>(*size) > sizeof(kTruncated)) {
       n = *size - sizeof(kTruncated);  // room for truncation message
     } else {
-      n = 0;                           // no room for truncation message
+      n = 0;  // no room for truncation message
     }
   }
   *size -= n;
@@ -105,9 +109,7 @@ inline static bool VADoRawLog(char** buf, int* size,
 }
 #endif  // ABSL_LOW_LEVEL_WRITE_SUPPORTED
 
-static constexpr int kLogBufSize = 3000;
-
-namespace {
+constexpr int kLogBufSize = 3000;
 
 // CAVEAT: vsnprintf called from *DoRawLog below has some (exotic) code paths
 // that invoke malloc() and getenv() that might acquire some locks.
@@ -166,7 +168,7 @@ void RawLogVA(absl::LogSeverity severity, const char* file, int line,
     } else {
       DoRawLog(&buf, &size, "%s", kTruncated);
     }
-    absl::raw_logging_internal::SafeWriteToStderr(buffer, strlen(buffer));
+    SafeWriteToStderr(buffer, strlen(buffer));
   }
 #else
   static_cast<void>(format);
@@ -181,11 +183,18 @@ void RawLogVA(absl::LogSeverity severity, const char* file, int line,
   }
 }
 
+// Non-formatting version of RawLog().
+//
+// TODO(gfalcon): When string_view no longer depends on base, change this
+// interface to take its message as a string_view instead.
+void DefaultInternalLog(absl::LogSeverity severity, const char* file, int line,
+                        const std::string& message) {
+  RawLog(severity, file, line, "%.*s", static_cast<int>(message.size()),
+         message.data());
+}
+
 }  // namespace
 
-namespace absl {
-ABSL_NAMESPACE_BEGIN
-namespace raw_logging_internal {
 void SafeWriteToStderr(const char *s, size_t len) {
 #if defined(ABSL_HAVE_SYSCALL_WRITE)
   syscall(SYS_write, STDERR_FILENO, s, len);
@@ -200,8 +209,6 @@ void SafeWriteToStderr(const char *s, size_t len) {
 #endif
 }
 
-void RawLog(absl::LogSeverity severity, const char* file, int line,
-            const char* format, ...) ABSL_PRINTF_ATTRIBUTE(4, 5);
 void RawLog(absl::LogSeverity severity, const char* file, int line,
             const char* format, ...) {
   va_list ap;
@@ -210,15 +217,6 @@ void RawLog(absl::LogSeverity severity, const char* file, int line,
   va_end(ap);
 }
 
-// Non-formatting version of RawLog().
-//
-// TODO(gfalcon): When string_view no longer depends on base, change this
-// interface to take its message as a string_view instead.
-static void DefaultInternalLog(absl::LogSeverity severity, const char* file,
-                               int line, const std::string& message) {
-  RawLog(severity, file, line, "%s", message.c_str());
-}
-
 bool RawLoggingFullySupported() {
 #ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED
   return true;
@@ -231,6 +229,10 @@ ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL
     absl::base_internal::AtomicHook<InternalLogFunction>
         internal_log_function(DefaultInternalLog);
 
+void RegisterLogPrefixHook(LogPrefixHook func) { log_prefix_hook.Store(func); }
+
+void RegisterAbortHook(AbortHook func) { abort_hook.Store(func); }
+
 void RegisterInternalLogFunction(InternalLogFunction func) {
   internal_log_function.Store(func);
 }

data/third_party/abseil-cpp/absl/base/internal/raw_logging.h

@@ -72,12 +72,14 @@
 //
 // The API is a subset of the above: each macro only takes two arguments.  Use
 // StrCat if you need to build a richer message.
-#define ABSL_INTERNAL_LOG(severity, message)                             \
-  do {                                                                   \
-    constexpr const char* absl_raw_logging_internal_filename = __FILE__; \
-    ::absl::raw_logging_internal::internal_log_function(                 \
-        ABSL_RAW_LOGGING_INTERNAL_##severity,                            \
-        absl_raw_logging_internal_filename, __LINE__, message);          \
+#define ABSL_INTERNAL_LOG(severity, message)                                 \
+  do {                                                                       \
+    constexpr const char* absl_raw_logging_internal_filename = __FILE__;     \
+    ::absl::raw_logging_internal::internal_log_function(                     \
+        ABSL_RAW_LOGGING_INTERNAL_##severity,                                \
+        absl_raw_logging_internal_filename, __LINE__, message);              \
+    if (ABSL_RAW_LOGGING_INTERNAL_##severity == ::absl::LogSeverity::kFatal) \
+      ABSL_INTERNAL_UNREACHABLE;                                             \
   } while (0)
 
 #define ABSL_INTERNAL_CHECK(condition, message)                    \
@@ -176,6 +178,14 @@ ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL extern base_internal::AtomicHook<
     InternalLogFunction>
     internal_log_function;
 
+// Registers hooks of the above types.  Only a single hook of each type may be
+// registered.  It is an error to call these functions multiple times with
+// different input arguments.
+//
+// These functions are safe to call at any point during initialization; they do
+// not block or malloc, and are async-signal safe.
+void RegisterLogPrefixHook(LogPrefixHook func);
+void RegisterAbortHook(AbortHook func);
 void RegisterInternalLogFunction(InternalLogFunction func);
 
 }  // namespace raw_logging_internal

data/third_party/abseil-cpp/absl/base/internal/spinlock.cc

@@ -125,8 +125,9 @@ void SpinLock::SlowLock() {
     // it as having a sleeper.
     if ((lock_value & kWaitTimeMask) == 0) {
       // Here, just "mark" that the thread is going to sleep.  Don't store the
-      // lock wait time in the lock as that will cause the current lock
-      // owner to think it experienced contention.
+      // lock wait time in the lock -- the lock word stores the amount of time
+      // that the current holder waited before acquiring the lock, not the wait
+      // time of any thread currently waiting to acquire it.
       if (lockword_.compare_exchange_strong(
               lock_value, lock_value | kSpinLockSleeper,
               std::memory_order_relaxed, std::memory_order_relaxed)) {
@@ -140,6 +141,14 @@ void SpinLock::SlowLock() {
         // this thread obtains the lock.
         lock_value = TryLockInternal(lock_value, wait_cycles);
         continue;   // Skip the delay at the end of the loop.
+      } else if ((lock_value & kWaitTimeMask) == 0) {
+        // The lock is still held, without a waiter being marked, but something
+        // else about the lock word changed, causing our CAS to fail. For
+        // example, a new lock holder may have acquired the lock with
+        // kSpinLockDisabledScheduling set, whereas the previous holder had not
+        // set that flag. In this case, attempt again to mark ourselves as a
+        // waiter.
+        continue;
       }
     }
 

data/third_party/abseil-cpp/absl/base/internal/spinlock.h

@@ -15,11 +15,8 @@
 //
 
 //  Most users requiring mutual exclusion should use Mutex.
-//  SpinLock is provided for use in three situations:
+//  SpinLock is provided for use in two situations:
 //   - for use in code that Mutex itself depends on
-//   - to get a faster fast-path release under low contention (without an
-//     atomic read-modify-write) In return, SpinLock has worse behaviour under
-//     contention, which is why Mutex is preferred in most situations.
 //   - for async signal safety (see below)
 
 // SpinLock is async signal safe.  If a spinlock is used within a signal
@@ -140,8 +137,20 @@ class ABSL_LOCKABLE SpinLock {
   //
   // bit[0] encodes whether a lock is being held.
   // bit[1] encodes whether a lock uses cooperative scheduling.
-  // bit[2] encodes whether a lock disables scheduling.
+  // bit[2] encodes whether the current lock holder disabled scheduling when
+  //        acquiring the lock. Only set when kSpinLockHeld is also set.
   // bit[3:31] encodes time a lock spent on waiting as a 29-bit unsigned int.
+  //        This is set by the lock holder to indicate how long it waited on
+  //        the lock before eventually acquiring it. The number of cycles is
+  //        encoded as a 29-bit unsigned int, or in the case that the current
+  //        holder did not wait but another waiter is queued, the LSB
+  //        (kSpinLockSleeper) is set. The implementation does not explicitly
+  //        track the number of queued waiters beyond this. It must always be
+  //        assumed that waiters may exist if the current holder was required to
+  //        queue.
+  //
+  // Invariant: if the lock is not held, the value is either 0 or
+  // kSpinLockCooperative.
   static constexpr uint32_t kSpinLockHeld = 1;
   static constexpr uint32_t kSpinLockCooperative = 2;
   static constexpr uint32_t kSpinLockDisabledScheduling = 4;

data/third_party/abseil-cpp/absl/base/internal/spinlock_akaros.inc

@@ -20,7 +20,7 @@
 
 extern "C" {
 
-ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay(
+ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)(
     std::atomic<uint32_t>* /* lock_word */, uint32_t /* value */,
     int /* loop */, absl::base_internal::SchedulingMode /* mode */) {
   // In Akaros, one must take care not to call anything that could cause a
@@ -29,7 +29,7 @@ ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay(
   // arbitrary code.
 }
 
-ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake(
+ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)(
     std::atomic<uint32_t>* /* lock_word */, bool /* all */) {}
 
 }  // extern "C"

data/third_party/abseil-cpp/absl/base/internal/spinlock_linux.inc

@@ -56,7 +56,7 @@ static_assert(sizeof(std::atomic<uint32_t>) == sizeof(int),
 
 extern "C" {
 
-ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay(
+ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)(
     std::atomic<uint32_t> *w, uint32_t value, int loop,
     absl::base_internal::SchedulingMode) {
   absl::base_internal::ErrnoSaver errno_saver;
@@ -66,8 +66,8 @@ ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay(
   syscall(SYS_futex, w, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, value, &tm);
 }
 
-ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake(std::atomic<uint32_t> *w,
-                                                  bool all) {
+ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)(
+    std::atomic<uint32_t> *w, bool all) {
   syscall(SYS_futex, w, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, all ? INT_MAX : 1, 0);
 }
 

data/third_party/abseil-cpp/absl/base/internal/spinlock_posix.inc

@@ -25,7 +25,7 @@
 
 extern "C" {
 
-ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay(
+ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)(
     std::atomic<uint32_t>* /* lock_word */, uint32_t /* value */, int loop,
     absl::base_internal::SchedulingMode /* mode */) {
   absl::base_internal::ErrnoSaver errno_saver;
@@ -40,7 +40,7 @@ ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay(
   }
 }
 
-ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake(
+ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)(
     std::atomic<uint32_t>* /* lock_word */, bool /* all */) {}
 
 }  // extern "C"

data/third_party/abseil-cpp/absl/base/internal/spinlock_wait.h

@@ -43,18 +43,16 @@ uint32_t SpinLockWait(std::atomic<uint32_t> *w, int n,
                       const SpinLockWaitTransition trans[],
                       SchedulingMode scheduling_mode);
 
-// If possible, wake some thread that has called SpinLockDelay(w, ...). If
-// "all" is true, wake all such threads.  This call is a hint, and on some
-// systems it may be a no-op; threads calling SpinLockDelay() will always wake
-// eventually even if SpinLockWake() is never called.
+// If possible, wake some thread that has called SpinLockDelay(w, ...). If `all`
+// is true, wake all such threads. On some systems, this may be a no-op; on
+// those systems, threads calling SpinLockDelay() will always wake eventually
+// even if SpinLockWake() is never called.
 void SpinLockWake(std::atomic<uint32_t> *w, bool all);
 
 // Wait for an appropriate spin delay on iteration "loop" of a
 // spin loop on location *w, whose previously observed value was "value".
 // SpinLockDelay() may do nothing, may yield the CPU, may sleep a clock tick,
-// or may wait for a delay that can be truncated by a call to SpinLockWake(w).
-// In all cases, it must return in bounded time even if SpinLockWake() is not
-// called.
+// or may wait for a call to SpinLockWake(w).
 void SpinLockDelay(std::atomic<uint32_t> *w, uint32_t value, int loop,
                    base_internal::SchedulingMode scheduling_mode);
 
@@ -73,21 +71,23 @@ ABSL_NAMESPACE_END
 // By changing our extension points to be extern "C", we dodge this
 // check.
 extern "C" {
-void AbslInternalSpinLockWake(std::atomic<uint32_t> *w, bool all);
-void AbslInternalSpinLockDelay(
+void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)(std::atomic<uint32_t> *w,
+                                                      bool all);
+void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)(
     std::atomic<uint32_t> *w, uint32_t value, int loop,
     absl::base_internal::SchedulingMode scheduling_mode);
 }
 
 inline void absl::base_internal::SpinLockWake(std::atomic<uint32_t> *w,
                                               bool all) {
-  AbslInternalSpinLockWake(w, all);
+  ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)(w, all);
 }
 
 inline void absl::base_internal::SpinLockDelay(
     std::atomic<uint32_t> *w, uint32_t value, int loop,
     absl::base_internal::SchedulingMode scheduling_mode) {
-  AbslInternalSpinLockDelay(w, value, loop, scheduling_mode);
+  ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)
+  (w, value, loop, scheduling_mode);
 }
 
 #endif  // ABSL_BASE_INTERNAL_SPINLOCK_WAIT_H_

data/third_party/abseil-cpp/absl/base/internal/spinlock_win32.inc

@@ -20,9 +20,9 @@
 
 extern "C" {
 
-void AbslInternalSpinLockDelay(std::atomic<uint32_t>* /* lock_word */,
-                               uint32_t /* value */, int loop,
-                               absl::base_internal::SchedulingMode /* mode */) {
+void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)(
+    std::atomic<uint32_t>* /* lock_word */, uint32_t /* value */, int loop,
+    absl::base_internal::SchedulingMode /* mode */) {
   if (loop == 0) {
   } else if (loop == 1) {
     Sleep(0);
@@ -31,7 +31,7 @@ void AbslInternalSpinLockDelay(std::atomic<uint32_t>* /* lock_word */,
   }
 }
 
-void AbslInternalSpinLockWake(std::atomic<uint32_t>* /* lock_word */,
-                              bool /* all */) {}
+void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)(
+    std::atomic<uint32_t>* /* lock_word */, bool /* all */) {}
 
 }  // extern "C"

data/third_party/abseil-cpp/absl/base/internal/sysinfo.cc

@@ -426,7 +426,7 @@ pid_t GetTID() {
 // userspace construct) to avoid unnecessary system calls. Without this caching,
 // it can take roughly 98ns, while it takes roughly 1ns with this caching.
 pid_t GetCachedTID() {
-#if ABSL_HAVE_THREAD_LOCAL
+#ifdef ABSL_HAVE_THREAD_LOCAL
   static thread_local pid_t thread_id = GetTID();
   return thread_id;
 #else

data/third_party/abseil-cpp/absl/base/internal/thread_identity.cc

@@ -23,6 +23,7 @@
 #include <cassert>
 #include <memory>
 
+#include "absl/base/attributes.h"
 #include "absl/base/call_once.h"
 #include "absl/base/internal/raw_logging.h"
 #include "absl/base/internal/spinlock.h"
@@ -53,9 +54,11 @@ void AllocateThreadIdentityKey(ThreadIdentityReclaimerFunction reclaimer) {
 // exist within a process (via dlopen() or similar), references to
 // thread_identity_ptr from each instance of the code will refer to
 // *different* instances of this ptr.
-#ifdef __GNUC__
+// Apple platforms have the visibility attribute, but issue a compile warning
+// that protected visibility is unsupported.
+#if ABSL_HAVE_ATTRIBUTE(visibility) && !defined(__APPLE__)
 __attribute__((visibility("protected")))
-#endif  // __GNUC__
+#endif  // ABSL_HAVE_ATTRIBUTE(visibility) && !defined(__APPLE__)
 #if ABSL_PER_THREAD_TLS
 // Prefer __thread to thread_local as benchmarks indicate it is a bit faster.
 ABSL_PER_THREAD_TLS_KEYWORD ThreadIdentity* thread_identity_ptr = nullptr;

data/third_party/abseil-cpp/absl/base/internal/thread_identity.h

@@ -32,6 +32,7 @@
 
 #include "absl/base/config.h"
 #include "absl/base/internal/per_thread_tls.h"
+#include "absl/base/optimization.h"
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
@@ -69,30 +70,28 @@ struct PerThreadSynch {
                          // is using this PerThreadSynch as a terminator.  Its
                          // skip field must not be filled in because the loop
                          // might then skip over the terminator.
-
-  // The wait parameters of the current wait.  waitp is null if the
-  // thread is not waiting. Transitions from null to non-null must
-  // occur before the enqueue commit point (state = kQueued in
-  // Enqueue() and CondVarEnqueue()). Transitions from non-null to
-  // null must occur after the wait is finished (state = kAvailable in
-  // Mutex::Block() and CondVar::WaitCommon()). This field may be
-  // changed only by the thread that describes this PerThreadSynch.  A
-  // special case is Fer(), which calls Enqueue() on another thread,
-  // but with an identical SynchWaitParams pointer, thus leaving the
-  // pointer unchanged.
-  SynchWaitParams *waitp;
-
-  bool suppress_fatal_errors;  // If true, try to proceed even in the face of
-                               // broken invariants.  This is used within fatal
-                               // signal handlers to improve the chances of
-                               // debug logging information being output
-                               // successfully.
-
-  intptr_t readers;     // Number of readers in mutex.
-  int priority;         // Priority of thread (updated every so often).
-
-  // When priority will next be read (cycles).
-  int64_t next_priority_read_cycles;
+  bool wake;             // This thread is to be woken from a Mutex.
+  // If "x" is on a waiter list for a mutex, "x->cond_waiter" is true iff the
+  // waiter is waiting on the mutex as part of a CV Wait or Mutex Await.
+  //
+  // The value of "x->cond_waiter" is meaningless if "x" is not on a
+  // Mutex waiter list.
+  bool cond_waiter;
+  bool maybe_unlocking;  // Valid at head of Mutex waiter queue;
+                         // true if UnlockSlow could be searching
+                         // for a waiter to wake.  Used for an optimization
+                         // in Enqueue().  true is always a valid value.
+                         // Can be reset to false when the unlocker or any
+                         // writer releases the lock, or a reader fully
+                         // releases the lock.  It may not be set to false
+                         // by a reader that decrements the count to
+                         // non-zero. protected by mutex spinlock
+  bool suppress_fatal_errors;  // If true, try to proceed even in the face
+                               // of broken invariants.  This is used within
+                               // fatal signal handlers to improve the
+                               // chances of debug logging information being
+                               // output successfully.
+  int priority;                // Priority of thread (updated every so often).
 
   // State values:
   //   kAvailable: This PerThreadSynch is available.
@@ -111,30 +110,30 @@ struct PerThreadSynch {
   };
   std::atomic<State> state;
 
-  bool maybe_unlocking;  // Valid at head of Mutex waiter queue;
-                         // true if UnlockSlow could be searching
-                         // for a waiter to wake.  Used for an optimization
-                         // in Enqueue().  true is always a valid value.
-                         // Can be reset to false when the unlocker or any
-                         // writer releases the lock, or a reader fully releases
-                         // the lock.  It may not be set to false by a reader
-                         // that decrements the count to non-zero.
-                         // protected by mutex spinlock
+  // The wait parameters of the current wait.  waitp is null if the
+  // thread is not waiting. Transitions from null to non-null must
+  // occur before the enqueue commit point (state = kQueued in
+  // Enqueue() and CondVarEnqueue()). Transitions from non-null to
+  // null must occur after the wait is finished (state = kAvailable in
+  // Mutex::Block() and CondVar::WaitCommon()). This field may be
+  // changed only by the thread that describes this PerThreadSynch.  A
+  // special case is Fer(), which calls Enqueue() on another thread,
+  // but with an identical SynchWaitParams pointer, thus leaving the
+  // pointer unchanged.
+  SynchWaitParams* waitp;
 
-  bool wake;  // This thread is to be woken from a Mutex.
+  intptr_t readers;     // Number of readers in mutex.
 
-  // If "x" is on a waiter list for a mutex, "x->cond_waiter" is true iff the
-  // waiter is waiting on the mutex as part of a CV Wait or Mutex Await.
-  //
-  // The value of "x->cond_waiter" is meaningless if "x" is not on a
-  // Mutex waiter list.
-  bool cond_waiter;
+  // When priority will next be read (cycles).
+  int64_t next_priority_read_cycles;
 
   // Locks held; used during deadlock detection.
   // Allocated in Synch_GetAllLocks() and freed in ReclaimThreadIdentity().
   SynchLocksHeld *all_locks;
 };
 
+// The instances of this class are allocated in NewThreadIdentity() with an
+// alignment of PerThreadSynch::kAlignment.
 struct ThreadIdentity {
   // Must be the first member.  The Mutex implementation requires that
   // the PerThreadSynch object associated with each thread is
@@ -144,7 +143,7 @@ struct ThreadIdentity {
 
   // Private: Reserved for absl::synchronization_internal::Waiter.
   struct WaiterState {
-    char data[128];
+    alignas(void*) char data[128];
   } waiter_state;
 
   // Used by PerThreadSem::{Get,Set}ThreadBlockedCounter().
@@ -212,7 +211,9 @@ void ClearCurrentThreadIdentity();
 #define ABSL_THREAD_IDENTITY_MODE ABSL_FORCE_THREAD_IDENTITY_MODE
 #elif defined(_WIN32) && !defined(__MINGW32__)
 #define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11
-#elif ABSL_PER_THREAD_TLS && defined(__GOOGLE_GRTE_VERSION__) && \
+#elif defined(__APPLE__) && defined(ABSL_HAVE_THREAD_LOCAL)
+#define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11
+#elif ABSL_PER_THREAD_TLS && defined(__GOOGLE_GRTE_VERSION__) &&        \
     (__GOOGLE_GRTE_VERSION__ >= 20140228L)
 // Support for async-safe TLS was specifically added in GRTEv4.  It's not
 // present in the upstream eglibc.