@nxtedition/rocksdb 15.2.2 → 15.2.4

package/deps/abseil-cpp/absl/synchronization/mutex.cc

@@ -1,2820 +0,0 @@
-// Copyright 2017 The Abseil Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "absl/synchronization/mutex.h"
-
-#ifdef _WIN32
-#include <windows.h>
-#ifdef ERROR
-#undef ERROR
-#endif
-#else
-#include <fcntl.h>
-#include <pthread.h>
-#include <sched.h>
-#include <sys/time.h>
-#endif
-
-#include <assert.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-
-#include <algorithm>
-#include <atomic>
-#include <cstddef>
-#include <cstdlib>
-#include <cstring>
-#include <thread>  // NOLINT(build/c++11)
-
-#include "absl/base/attributes.h"
-#include "absl/base/call_once.h"
-#include "absl/base/config.h"
-#include "absl/base/dynamic_annotations.h"
-#include "absl/base/internal/atomic_hook.h"
-#include "absl/base/internal/cycleclock.h"
-#include "absl/base/internal/hide_ptr.h"
-#include "absl/base/internal/low_level_alloc.h"
-#include "absl/base/internal/raw_logging.h"
-#include "absl/base/internal/spinlock.h"
-#include "absl/base/internal/sysinfo.h"
-#include "absl/base/internal/thread_identity.h"
-#include "absl/base/internal/tsan_mutex_interface.h"
-#include "absl/base/optimization.h"
-#include "absl/debugging/stacktrace.h"
-#include "absl/debugging/symbolize.h"
-#include "absl/synchronization/internal/graphcycles.h"
-#include "absl/synchronization/internal/per_thread_sem.h"
-#include "absl/time/time.h"
-
-using absl::base_internal::CurrentThreadIdentityIfPresent;
-using absl::base_internal::CycleClock;
-using absl::base_internal::PerThreadSynch;
-using absl::base_internal::SchedulingGuard;
-using absl::base_internal::ThreadIdentity;
-using absl::synchronization_internal::GetOrCreateCurrentThreadIdentity;
-using absl::synchronization_internal::GraphCycles;
-using absl::synchronization_internal::GraphId;
-using absl::synchronization_internal::InvalidGraphId;
-using absl::synchronization_internal::KernelTimeout;
-using absl::synchronization_internal::PerThreadSem;
-
-extern "C" {
-ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalMutexYield)() {
-  std::this_thread::yield();
-}
-}  // extern "C"
-
-namespace absl {
-ABSL_NAMESPACE_BEGIN
-
-namespace {
-
-#if defined(ABSL_HAVE_THREAD_SANITIZER)
-constexpr OnDeadlockCycle kDeadlockDetectionDefault = OnDeadlockCycle::kIgnore;
-#else
-constexpr OnDeadlockCycle kDeadlockDetectionDefault = OnDeadlockCycle::kAbort;
-#endif
-
-ABSL_CONST_INIT std::atomic<OnDeadlockCycle> synch_deadlock_detection(
-    kDeadlockDetectionDefault);
-ABSL_CONST_INIT std::atomic<bool> synch_check_invariants(false);
-
-ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES
-absl::base_internal::AtomicHook<void (*)(int64_t wait_cycles)>
-    submit_profile_data;
-ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES absl::base_internal::AtomicHook<void (*)(
-    const char* msg, const void* obj, int64_t wait_cycles)>
-    mutex_tracer;
-ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES
-absl::base_internal::AtomicHook<void (*)(const char* msg, const void* cv)>
-    cond_var_tracer;
-
-}  // namespace
-
-static inline bool EvalConditionAnnotated(const Condition* cond, Mutex* mu,
-                                          bool locking, bool trylock,
-                                          bool read_lock);
-
-void RegisterMutexProfiler(void (*fn)(int64_t wait_cycles)) {
-  submit_profile_data.Store(fn);
-}
-
-void RegisterMutexTracer(void (*fn)(const char* msg, const void* obj,
-                                    int64_t wait_cycles)) {
-  mutex_tracer.Store(fn);
-}
-
-void RegisterCondVarTracer(void (*fn)(const char* msg, const void* cv)) {
-  cond_var_tracer.Store(fn);
-}
-
-namespace {
-// Represents the strategy for spin and yield.
-// See the comment in GetMutexGlobals() for more information.
-enum DelayMode { AGGRESSIVE, GENTLE };
-
-struct ABSL_CACHELINE_ALIGNED MutexGlobals {
-  absl::once_flag once;
-  // Note: this variable is initialized separately in Mutex::LockSlow,
-  // so that Mutex::Lock does not have a stack frame in optimized build.
-  std::atomic<int> spinloop_iterations{0};
-  int32_t mutex_sleep_spins[2] = {};
-  absl::Duration mutex_sleep_time;
-};
-
-ABSL_CONST_INIT static MutexGlobals globals;
-
-absl::Duration MeasureTimeToYield() {
-  absl::Time before = absl::Now();
-  ABSL_INTERNAL_C_SYMBOL(AbslInternalMutexYield)();
-  return absl::Now() - before;
-}
-
-const MutexGlobals& GetMutexGlobals() {
-  absl::base_internal::LowLevelCallOnce(&globals.once, [&]() {
-    if (absl::base_internal::NumCPUs() > 1) {
-      // If the mode is aggressive then spin many times before yielding.
-      // If the mode is gentle then spin only a few times before yielding.
-      // Aggressive spinning is used to ensure that an Unlock() call,
-      // which must get the spin lock for any thread to make progress gets it
-      // without undue delay.
-      globals.mutex_sleep_spins[AGGRESSIVE] = 5000;
-      globals.mutex_sleep_spins[GENTLE] = 250;
-      globals.mutex_sleep_time = absl::Microseconds(10);
-    } else {
-      // If this a uniprocessor, only yield/sleep. Real-time threads are often
-      // unable to yield, so the sleep time needs to be long enough to keep
-      // the calling thread asleep until scheduling happens.
-      globals.mutex_sleep_spins[AGGRESSIVE] = 0;
-      globals.mutex_sleep_spins[GENTLE] = 0;
-      globals.mutex_sleep_time = MeasureTimeToYield() * 5;
-      globals.mutex_sleep_time =
-          std::min(globals.mutex_sleep_time, absl::Milliseconds(1));
-      globals.mutex_sleep_time =
-          std::max(globals.mutex_sleep_time, absl::Microseconds(10));
-    }
-  });
-  return globals;
-}
-}  // namespace
-
-namespace synchronization_internal {
-// Returns the Mutex delay on iteration `c` depending on the given `mode`.
-// The returned value should be used as `c` for the next call to `MutexDelay`.
-int MutexDelay(int32_t c, int mode) {
-  const int32_t limit = GetMutexGlobals().mutex_sleep_spins[mode];
-  const absl::Duration sleep_time = GetMutexGlobals().mutex_sleep_time;
-  if (c < limit) {
-    // Spin.
-    c++;
-  } else {
-    SchedulingGuard::ScopedEnable enable_rescheduling;
-    ABSL_TSAN_MUTEX_PRE_DIVERT(nullptr, 0);
-    if (c == limit) {
-      // Yield once.
-      ABSL_INTERNAL_C_SYMBOL(AbslInternalMutexYield)();
-      c++;
-    } else {
-      // Then wait.
-      absl::SleepFor(sleep_time);
-      c = 0;
-    }
-    ABSL_TSAN_MUTEX_POST_DIVERT(nullptr, 0);
-  }
-  return c;
-}
-}  // namespace synchronization_internal
-
-// --------------------------Generic atomic ops
-// Ensure that "(*pv & bits) == bits" by doing an atomic update of "*pv" to
-// "*pv | bits" if necessary.  Wait until (*pv & wait_until_clear)==0
-// before making any change.
-// Returns true if bits were previously unset and set by the call.
-// This is used to set flags in mutex and condition variable words.
-static bool AtomicSetBits(std::atomic<intptr_t>* pv, intptr_t bits,
-                          intptr_t wait_until_clear) {
-  for (;;) {
-    intptr_t v = pv->load(std::memory_order_relaxed);
-    if ((v & bits) == bits) {
-      return false;
-    }
-    if ((v & wait_until_clear) != 0) {
-      continue;
-    }
-    if (pv->compare_exchange_weak(v, v | bits, std::memory_order_release,
-                                  std::memory_order_relaxed)) {
-      return true;
-    }
-  }
-}
-
-//------------------------------------------------------------------
-
-// Data for doing deadlock detection.
-ABSL_CONST_INIT static absl::base_internal::SpinLock deadlock_graph_mu(
-    base_internal::SCHEDULE_KERNEL_ONLY);
-
-// Graph used to detect deadlocks.
-ABSL_CONST_INIT static GraphCycles* deadlock_graph
-    ABSL_GUARDED_BY(deadlock_graph_mu) ABSL_PT_GUARDED_BY(deadlock_graph_mu);
-
-//------------------------------------------------------------------
-// An event mechanism for debugging mutex use.
-// It also allows mutexes to be given names for those who can't handle
-// addresses, and instead like to give their data structures names like
-// "Henry", "Fido", or "Rupert IV, King of Yondavia".
-
-namespace {  // to prevent name pollution
-enum {       // Mutex and CondVar events passed as "ev" to PostSynchEvent
-             // Mutex events
-  SYNCH_EV_TRYLOCK_SUCCESS,
-  SYNCH_EV_TRYLOCK_FAILED,
-  SYNCH_EV_READERTRYLOCK_SUCCESS,
-  SYNCH_EV_READERTRYLOCK_FAILED,
-  SYNCH_EV_LOCK,
-  SYNCH_EV_LOCK_RETURNING,
-  SYNCH_EV_READERLOCK,
-  SYNCH_EV_READERLOCK_RETURNING,
-  SYNCH_EV_UNLOCK,
-  SYNCH_EV_READERUNLOCK,
-
-  // CondVar events
-  SYNCH_EV_WAIT,
-  SYNCH_EV_WAIT_RETURNING,
-  SYNCH_EV_SIGNAL,
-  SYNCH_EV_SIGNALALL,
-};
-
-enum {                    // Event flags
-  SYNCH_F_R = 0x01,       // reader event
-  SYNCH_F_LCK = 0x02,     // PostSynchEvent called with mutex held
-  SYNCH_F_TRY = 0x04,     // TryLock or ReaderTryLock
-  SYNCH_F_UNLOCK = 0x08,  // Unlock or ReaderUnlock
-
-  SYNCH_F_LCK_W = SYNCH_F_LCK,
-  SYNCH_F_LCK_R = SYNCH_F_LCK | SYNCH_F_R,
-};
-}  // anonymous namespace
-
-// Properties of the events.
-static const struct {
-  int flags;
-  const char* msg;
-} event_properties[] = {
-    {SYNCH_F_LCK_W | SYNCH_F_TRY, "TryLock succeeded "},
-    {0, "TryLock failed "},
-    {SYNCH_F_LCK_R | SYNCH_F_TRY, "ReaderTryLock succeeded "},
-    {0, "ReaderTryLock failed "},
-    {0, "Lock blocking "},
-    {SYNCH_F_LCK_W, "Lock returning "},
-    {0, "ReaderLock blocking "},
-    {SYNCH_F_LCK_R, "ReaderLock returning "},
-    {SYNCH_F_LCK_W | SYNCH_F_UNLOCK, "Unlock "},
-    {SYNCH_F_LCK_R | SYNCH_F_UNLOCK, "ReaderUnlock "},
-    {0, "Wait on "},
-    {0, "Wait unblocked "},
-    {0, "Signal on "},
-    {0, "SignalAll on "},
-};
-
-ABSL_CONST_INIT static absl::base_internal::SpinLock synch_event_mu(
-    base_internal::SCHEDULE_KERNEL_ONLY);
-
-// Hash table size; should be prime > 2.
-// Can't be too small, as it's used for deadlock detection information.
-static constexpr uint32_t kNSynchEvent = 1031;
-
-static struct SynchEvent {  // this is a trivial hash table for the events
-  // struct is freed when refcount reaches 0
-  int refcount ABSL_GUARDED_BY(synch_event_mu);
-
-  // buckets have linear, 0-terminated  chains
-  SynchEvent* next ABSL_GUARDED_BY(synch_event_mu);
-
-  // Constant after initialization
-  uintptr_t masked_addr;  // object at this address is called "name"
-
-  // No explicit synchronization used.  Instead we assume that the
-  // client who enables/disables invariants/logging on a Mutex does so
-  // while the Mutex is not being concurrently accessed by others.
-  void (*invariant)(void* arg);  // called on each event
-  void* arg;                     // first arg to (*invariant)()
-  bool log;                      // logging turned on
-
-  // Constant after initialization
-  char name[1];  // actually longer---NUL-terminated string
-}* synch_event[kNSynchEvent] ABSL_GUARDED_BY(synch_event_mu);
-
-// Ensure that the object at "addr" has a SynchEvent struct associated with it,
-// set "bits" in the word there (waiting until lockbit is clear before doing
-// so), and return a refcounted reference that will remain valid until
-// UnrefSynchEvent() is called.  If a new SynchEvent is allocated,
-// the string name is copied into it.
-// When used with a mutex, the caller should also ensure that kMuEvent
-// is set in the mutex word, and similarly for condition variables and kCVEvent.
-static SynchEvent* EnsureSynchEvent(std::atomic<intptr_t>* addr,
-                                    const char* name, intptr_t bits,
-                                    intptr_t lockbit) {
-  uint32_t h = reinterpret_cast<uintptr_t>(addr) % kNSynchEvent;
-  synch_event_mu.lock();
-  // When a Mutex/CondVar is destroyed, we don't remove the associated
-  // SynchEvent to keep destructors empty in release builds for performance
-  // reasons. If the current call is the first to set bits (kMuEvent/kCVEvent),
-  // we don't look up the existing even because (if it exists, it must be for
-  // the previous Mutex/CondVar that existed at the same address).
-  // The leaking events must not be a problem for tests, which should create
-  // bounded amount of events. And debug logging is not supposed to be enabled
-  // in production. However, if it's accidentally enabled, or briefly enabled
-  // for some debugging, we don't want to crash the program. Instead we drop
-  // all events, if we accumulated too many of them. Size of a single event
-  // is ~48 bytes, so 100K events is ~5 MB.
-  // Additionally we could delete the old event for the same address,
-  // but it would require a better hashmap (if we accumulate too many events,
-  // linked lists will grow and traversing them will be very slow).
-  constexpr size_t kMaxSynchEventCount = 100 << 10;
-  // Total number of live synch events.
-  static size_t synch_event_count ABSL_GUARDED_BY(synch_event_mu);
-  if (++synch_event_count > kMaxSynchEventCount) {
-    synch_event_count = 0;
-    ABSL_RAW_LOG(ERROR,
-                 "Accumulated %zu Mutex debug objects. If you see this"
-                 " in production, it may mean that the production code"
-                 " accidentally calls "
-                 "Mutex/CondVar::EnableDebugLog/EnableInvariantDebugging.",
-                 kMaxSynchEventCount);
-    for (auto*& head : synch_event) {
-      for (auto* e = head; e != nullptr;) {
-        SynchEvent* next = e->next;
-        if (--(e->refcount) == 0) {
-          base_internal::LowLevelAlloc::Free(e);
-        }
-        e = next;
-      }
-      head = nullptr;
-    }
-  }
-  SynchEvent* e = nullptr;
-  if (!AtomicSetBits(addr, bits, lockbit)) {
-    for (e = synch_event[h];
-         e != nullptr && e->masked_addr != base_internal::HidePtr(addr);
-         e = e->next) {
-    }
-  }
-  if (e == nullptr) {  // no SynchEvent struct found; make one.
-    if (name == nullptr) {
-      name = "";
-    }
-    size_t l = strlen(name);
-    e = reinterpret_cast<SynchEvent*>(
-        base_internal::LowLevelAlloc::Alloc(sizeof(*e) + l));
-    e->refcount = 2;  // one for return value, one for linked list
-    e->masked_addr = base_internal::HidePtr(addr);
-    e->invariant = nullptr;
-    e->arg = nullptr;
-    e->log = false;
-    strcpy(e->name, name);  // NOLINT(runtime/printf)
-    e->next = synch_event[h];
-    synch_event[h] = e;
-  } else {
-    e->refcount++;  // for return value
-  }
-  synch_event_mu.unlock();
-  return e;
-}
-
-// Decrement the reference count of *e, or do nothing if e==null.
-static void UnrefSynchEvent(SynchEvent* e) {
-  if (e != nullptr) {
-    synch_event_mu.lock();
-    bool del = (--(e->refcount) == 0);
-    synch_event_mu.unlock();
-    if (del) {
-      base_internal::LowLevelAlloc::Free(e);
-    }
-  }
-}
-
-// Return a refcounted reference to the SynchEvent of the object at address
-// "addr", if any.  The pointer returned is valid until the UnrefSynchEvent() is
-// called.
-static SynchEvent* GetSynchEvent(const void* addr) {
-  uint32_t h = reinterpret_cast<uintptr_t>(addr) % kNSynchEvent;
-  SynchEvent* e;
-  synch_event_mu.lock();
-  for (e = synch_event[h];
-       e != nullptr && e->masked_addr != base_internal::HidePtr(addr);
-       e = e->next) {
-  }
-  if (e != nullptr) {
-    e->refcount++;
-  }
-  synch_event_mu.unlock();
-  return e;
-}
-
-// Called when an event "ev" occurs on a Mutex of CondVar "obj"
-// if event recording is on
-static void PostSynchEvent(void* obj, int ev) {
-  SynchEvent* e = GetSynchEvent(obj);
-  // logging is on if event recording is on and either there's no event struct,
-  // or it explicitly says to log
-  if (e == nullptr || e->log) {
-    void* pcs[40];
-    int n = absl::GetStackTrace(pcs, ABSL_ARRAYSIZE(pcs), 1);
-    // A buffer with enough space for the ASCII for all the PCs, even on a
-    // 64-bit machine.
-    char buffer[ABSL_ARRAYSIZE(pcs) * 24];
-    int pos = snprintf(buffer, sizeof(buffer), " @");
-    for (int i = 0; i != n; i++) {
-      int b = snprintf(&buffer[pos], sizeof(buffer) - static_cast<size_t>(pos),
-                       " %p", pcs[i]);
-      if (b < 0 ||
-          static_cast<size_t>(b) >= sizeof(buffer) - static_cast<size_t>(pos)) {
-        break;
-      }
-      pos += b;
-    }
-    ABSL_RAW_LOG(INFO, "%s%p %s %s", event_properties[ev].msg, obj,
-                 (e == nullptr ? "" : e->name), buffer);
-  }
-  const int flags = event_properties[ev].flags;
-  if ((flags & SYNCH_F_LCK) != 0 && e != nullptr && e->invariant != nullptr) {
-    // Calling the invariant as is causes problems under ThreadSanitizer.
-    // We are currently inside of Mutex Lock/Unlock and are ignoring all
-    // memory accesses and synchronization. If the invariant transitively
-    // synchronizes something else and we ignore the synchronization, we will
-    // get false positive race reports later.
-    // Reuse EvalConditionAnnotated to properly call into user code.
-    struct local {
-      static bool pred(SynchEvent* ev) {
-        (*ev->invariant)(ev->arg);
-        return false;
-      }
-    };
-    Condition cond(&local::pred, e);
-    Mutex* mu = static_cast<Mutex*>(obj);
-    const bool locking = (flags & SYNCH_F_UNLOCK) == 0;
-    const bool trylock = (flags & SYNCH_F_TRY) != 0;
-    const bool read_lock = (flags & SYNCH_F_R) != 0;
-    EvalConditionAnnotated(&cond, mu, locking, trylock, read_lock);
-  }
-  UnrefSynchEvent(e);
-}
-
-//------------------------------------------------------------------
-
-// The SynchWaitParams struct encapsulates the way in which a thread is waiting:
-// whether it has a timeout, the condition, exclusive/shared, and whether a
-// condition variable wait has an associated Mutex (as opposed to another
-// type of lock).  It also points to the PerThreadSynch struct of its thread.
-// cv_word tells Enqueue() to enqueue on a CondVar using CondVarEnqueue().
-//
-// This structure is held on the stack rather than directly in
-// PerThreadSynch because a thread can be waiting on multiple Mutexes if,
-// while waiting on one Mutex, the implementation calls a client callback
-// (such as a Condition function) that acquires another Mutex. We don't
-// strictly need to allow this, but programmers become confused if we do not
-// allow them to use functions such a LOG() within Condition functions.  The
-// PerThreadSynch struct points at the most recent SynchWaitParams struct when
-// the thread is on a Mutex's waiter queue.
-struct SynchWaitParams {
-  SynchWaitParams(Mutex::MuHow how_arg, const Condition* cond_arg,
-                  KernelTimeout timeout_arg, Mutex* cvmu_arg,
-                  PerThreadSynch* thread_arg,
-                  std::atomic<intptr_t>* cv_word_arg)
-      : how(how_arg),
-        cond(cond_arg),
-        timeout(timeout_arg),
-        cvmu(cvmu_arg),
-        thread(thread_arg),
-        cv_word(cv_word_arg),
-        contention_start_cycles(CycleClock::Now()),
-        should_submit_contention_data(false) {}
-
-  const Mutex::MuHow how;  // How this thread needs to wait.
-  const Condition* cond;   // The condition that this thread is waiting for.
-                           // In Mutex, this field is set to zero if a timeout
-                           // expires.
-  KernelTimeout timeout;   // timeout expiry---absolute time
-                           // In Mutex, this field is set to zero if a timeout
-                           // expires.
-  Mutex* const cvmu;       // used for transfer from cond var to mutex
-  PerThreadSynch* const thread;  // thread that is waiting
-
-  // If not null, thread should be enqueued on the CondVar whose state
-  // word is cv_word instead of queueing normally on the Mutex.
-  std::atomic<intptr_t>* cv_word;
-
-  int64_t contention_start_cycles;  // Time (in cycles) when this thread started
-                                    // to contend for the mutex.
-  bool should_submit_contention_data;
-};
-
-struct SynchLocksHeld {
-  int n;          // number of valid entries in locks[]
-  bool overflow;  // true iff we overflowed the array at some point
-  struct {
-    Mutex* mu;      // lock acquired
-    int32_t count;  // times acquired
-    GraphId id;     // deadlock_graph id of acquired lock
-  } locks[40];
-  // If a thread overfills the array during deadlock detection, we
-  // continue, discarding information as needed.  If no overflow has
-  // taken place, we can provide more error checking, such as
-  // detecting when a thread releases a lock it does not hold.
-};
-
-// A sentinel value in lists that is not 0.
-// A 0 value is used to mean "not on a list".
-static PerThreadSynch* const kPerThreadSynchNull =
-    reinterpret_cast<PerThreadSynch*>(1);
-
-static SynchLocksHeld* LocksHeldAlloc() {
-  SynchLocksHeld* ret = reinterpret_cast<SynchLocksHeld*>(
-      base_internal::LowLevelAlloc::Alloc(sizeof(SynchLocksHeld)));
-  ret->n = 0;
-  ret->overflow = false;
-  return ret;
-}
-
-// Return the PerThreadSynch-struct for this thread.
-static PerThreadSynch* Synch_GetPerThread() {
-  ThreadIdentity* identity = GetOrCreateCurrentThreadIdentity();
-  return &identity->per_thread_synch;
-}
-
-static PerThreadSynch* Synch_GetPerThreadAnnotated(Mutex* mu) {
-  if (mu) {
-    ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0);
-  }
-  PerThreadSynch* w = Synch_GetPerThread();
-  if (mu) {
-    ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0);
-  }
-  return w;
-}
-
-static SynchLocksHeld* Synch_GetAllLocks() {
-  PerThreadSynch* s = Synch_GetPerThread();
-  if (s->all_locks == nullptr) {
-    s->all_locks = LocksHeldAlloc();  // Freed by ReclaimThreadIdentity.
-  }
-  return s->all_locks;
-}
-
-// Post on "w"'s associated PerThreadSem.
-void Mutex::IncrementSynchSem(Mutex* mu, PerThreadSynch* w) {
-  static_cast<void>(mu);  // Prevent unused param warning in non-TSAN builds.
-  ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0);
-  // We miss synchronization around passing PerThreadSynch between threads
-  // since it happens inside of the Mutex code, so we need to ignore all
-  // accesses to the object.
-  ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN();
-  PerThreadSem::Post(w->thread_identity());
-  ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_END();
-  ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0);
-}
-
-// Wait on "w"'s associated PerThreadSem; returns false if timeout expired.
-bool Mutex::DecrementSynchSem(Mutex* mu, PerThreadSynch* w, KernelTimeout t) {
-  static_cast<void>(mu);  // Prevent unused param warning in non-TSAN builds.
-  ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0);
-  assert(w == Synch_GetPerThread());
-  static_cast<void>(w);
-  bool res = PerThreadSem::Wait(t);
-  ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0);
-  return res;
-}
-
-// We're in a fatal signal handler that hopes to use Mutex and to get
-// lucky by not deadlocking.  We try to improve its chances of success
-// by effectively disabling some of the consistency checks.  This will
-// prevent certain ABSL_RAW_CHECK() statements from being triggered when
-// re-rentry is detected.  The ABSL_RAW_CHECK() statements are those in the
-// Mutex code checking that the "waitp" field has not been reused.
-void Mutex::InternalAttemptToUseMutexInFatalSignalHandler() {
-  // Fix the per-thread state only if it exists.
-  ThreadIdentity* identity = CurrentThreadIdentityIfPresent();
-  if (identity != nullptr) {
-    identity->per_thread_synch.suppress_fatal_errors = true;
-  }
-  // Don't do deadlock detection when we are already failing.
-  synch_deadlock_detection.store(OnDeadlockCycle::kIgnore,
-                                 std::memory_order_release);
-}
-
-// --------------------------Mutexes
-
-// In the layout below, the msb of the bottom byte is currently unused.  Also,
-// the following constraints were considered in choosing the layout:
-//  o Both the debug allocator's "uninitialized" and "freed" patterns (0xab and
-//    0xcd) are illegal: reader and writer lock both held.
-//  o kMuWriter and kMuEvent should exceed kMuDesig and kMuWait, to enable the
-//    bit-twiddling trick in Mutex::Unlock().
-//  o kMuWriter / kMuReader == kMuWrWait / kMuWait,
-//    to enable the bit-twiddling trick in CheckForMutexCorruption().
-static const intptr_t kMuReader = 0x0001L;  // a reader holds the lock
-// There's a designated waker.
-// INVARIANT1:  there's a thread that was blocked on the mutex, is
-// no longer, yet has not yet acquired the mutex.  If there's a
-// designated waker, all threads can avoid taking the slow path in
-// unlock because the designated waker will subsequently acquire
-// the lock and wake someone.  To maintain INVARIANT1 the bit is
-// set when a thread is unblocked(INV1a), and threads that were
-// unblocked reset the bit when they either acquire or re-block (INV1b).
-static const intptr_t kMuDesig = 0x0002L;
-static const intptr_t kMuWait = 0x0004L;    // threads are waiting
-static const intptr_t kMuWriter = 0x0008L;  // a writer holds the lock
-static const intptr_t kMuEvent = 0x0010L;   // record this mutex's events
-// Runnable writer is waiting for a reader.
-// If set, new readers will not lock the mutex to avoid writer starvation.
-// Note: if a reader has higher priority than the writer, it will still lock
-// the mutex ahead of the waiting writer, but in a very inefficient manner:
-// the reader will first queue itself and block, but then the last unlocking
-// reader will wake it.
-static const intptr_t kMuWrWait = 0x0020L;
-static const intptr_t kMuSpin = 0x0040L;  // spinlock protects wait list
-static const intptr_t kMuLow = 0x00ffL;   // mask all mutex bits
-static const intptr_t kMuHigh = ~kMuLow;  // mask pointer/reader count
-
-static_assert((0xab & (kMuWriter | kMuReader)) == (kMuWriter | kMuReader),
-              "The debug allocator's uninitialized pattern (0xab) must be an "
-              "invalid mutex state");
-static_assert((0xcd & (kMuWriter | kMuReader)) == (kMuWriter | kMuReader),
-              "The debug allocator's freed pattern (0xcd) must be an invalid "
-              "mutex state");
-
-// Hack to make constant values available to gdb pretty printer
-enum {
-  kGdbMuSpin = kMuSpin,
-  kGdbMuEvent = kMuEvent,
-  kGdbMuWait = kMuWait,
-  kGdbMuWriter = kMuWriter,
-  kGdbMuDesig = kMuDesig,
-  kGdbMuWrWait = kMuWrWait,
-  kGdbMuReader = kMuReader,
-  kGdbMuLow = kMuLow,
-};
-
-// kMuWrWait implies kMuWait.
-// kMuReader and kMuWriter are mutually exclusive.
-// If kMuReader is zero, there are no readers.
-// Otherwise, if kMuWait is zero, the high order bits contain a count of the
-// number of readers.  Otherwise, the reader count is held in
-// PerThreadSynch::readers of the most recently queued waiter, again in the
-// bits above kMuLow.
-static const intptr_t kMuOne = 0x0100;  // a count of one reader
-
-// flags passed to Enqueue and LockSlow{,WithTimeout,Loop}
-static const int kMuHasBlocked = 0x01;  // already blocked (MUST == 1)
-static const int kMuIsCond = 0x02;      // conditional waiter (CV or Condition)
-static const int kMuIsFer = 0x04;       // wait morphing from a CondVar
-
-static_assert(PerThreadSynch::kAlignment > kMuLow,
-              "PerThreadSynch::kAlignment must be greater than kMuLow");
-
-// This struct contains various bitmasks to be used in
-// acquiring and releasing a mutex in a particular mode.
-struct MuHowS {
-  // if all the bits in fast_need_zero are zero, the lock can be acquired by
-  // adding fast_add and oring fast_or.  The bit kMuDesig should be reset iff
-  // this is the designated waker.
-  intptr_t fast_need_zero;
-  intptr_t fast_or;
-  intptr_t fast_add;
-
-  intptr_t slow_need_zero;  // fast_need_zero with events (e.g. logging)
-
-  intptr_t slow_inc_need_zero;  // if all the bits in slow_inc_need_zero are
-                                // zero a reader can acquire a read share by
-                                // setting the reader bit and incrementing
-                                // the reader count (in last waiter since
-                                // we're now slow-path).  kMuWrWait be may
-                                // be ignored if we already waited once.
-};
-
-static const MuHowS kSharedS = {
-    // shared or read lock
-    kMuWriter | kMuWait | kMuEvent,   // fast_need_zero
-    kMuReader,                        // fast_or
-    kMuOne,                           // fast_add
-    kMuWriter | kMuWait,              // slow_need_zero
-    kMuSpin | kMuWriter | kMuWrWait,  // slow_inc_need_zero
-};
-static const MuHowS kExclusiveS = {
-    // exclusive or write lock
-    kMuWriter | kMuReader | kMuEvent,  // fast_need_zero
-    kMuWriter,                         // fast_or
-    0,                                 // fast_add
-    kMuWriter | kMuReader,             // slow_need_zero
-    ~static_cast<intptr_t>(0),         // slow_inc_need_zero
-};
-static const Mutex::MuHow kShared = &kSharedS;        // shared lock
-static const Mutex::MuHow kExclusive = &kExclusiveS;  // exclusive lock
-
-#ifdef NDEBUG
-static constexpr bool kDebugMode = false;
-#else
-static constexpr bool kDebugMode = true;
-#endif
-
-#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE
-static unsigned TsanFlags(Mutex::MuHow how) {
-  return how == kShared ? __tsan_mutex_read_lock : 0;
-}
-#endif
-
-#if defined(__APPLE__) || defined(ABSL_BUILD_DLL)
-// When building a dll symbol export lists may reference the destructor
-// and want it to be an exported symbol rather than an inline function.
-// Some apple builds also do dynamic library build but don't say it explicitly.
-Mutex::~Mutex() { Dtor(); }
-#endif
-
-#if !defined(NDEBUG) || defined(ABSL_HAVE_THREAD_SANITIZER) || \
-    defined(ABSL_BUILD_DLL)
-void Mutex::Dtor() {
-  if (kDebugMode) {
-    this->ForgetDeadlockInfo();
-  }
-  ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static);
-}
-#endif
-
-void Mutex::EnableDebugLog(const char* name) {
-  // Need to disable writes here and in EnableInvariantDebugging to prevent
-  // false race reports on SynchEvent objects. TSan ignores synchronization
-  // on synch_event_mu in Lock/Unlock/etc methods due to mutex annotations,
-  // but it sees few accesses to SynchEvent in EvalConditionAnnotated.
-  // If we don't ignore accesses here, it can result in false races
-  // between EvalConditionAnnotated and SynchEvent reuse in EnsureSynchEvent.
-  ABSL_ANNOTATE_IGNORE_WRITES_BEGIN();
-  SynchEvent* e = EnsureSynchEvent(&this->mu_, name, kMuEvent, kMuSpin);
-  e->log = true;
-  UnrefSynchEvent(e);
-  // This prevents "error: undefined symbol: absl::Mutex::~Mutex()"
-  // in a release build (NDEBUG defined) when a test does "#undef NDEBUG"
-  // to use assert macro. In such case, the test does not get the dtor
-  // definition because it's supposed to be outline when NDEBUG is not defined,
-  // and this source file does not define one either because NDEBUG is defined.
-  // Since it's not possible to take address of a destructor, we move the
-  // actual destructor code into the separate Dtor function and force the
-  // compiler to emit this function even if it's inline by taking its address.
-  ABSL_ATTRIBUTE_UNUSED volatile auto dtor = &Mutex::Dtor;
-  ABSL_ANNOTATE_IGNORE_WRITES_END();
-}
-
-void EnableMutexInvariantDebugging(bool enabled) {
-  synch_check_invariants.store(enabled, std::memory_order_release);
-}
-
-void Mutex::EnableInvariantDebugging(void (*invariant)(void*), void* arg) {
-  ABSL_ANNOTATE_IGNORE_WRITES_BEGIN();
-  if (synch_check_invariants.load(std::memory_order_acquire) &&
-      invariant != nullptr) {
-    SynchEvent* e = EnsureSynchEvent(&this->mu_, nullptr, kMuEvent, kMuSpin);
-    e->invariant = invariant;
-    e->arg = arg;
-    UnrefSynchEvent(e);
-  }
-  ABSL_ANNOTATE_IGNORE_WRITES_END();
-}
-
-void SetMutexDeadlockDetectionMode(OnDeadlockCycle mode) {
-  synch_deadlock_detection.store(mode, std::memory_order_release);
-}
-
-// Return true iff threads x and y are part of the same equivalence
-// class of waiters. An equivalence class is defined as the set of
-// waiters with the same condition, type of lock, and thread priority.
-//
-// Requires that x and y be waiting on the same Mutex queue.
-static bool MuEquivalentWaiter(PerThreadSynch* x, PerThreadSynch* y) {
-  return x->waitp->how == y->waitp->how && x->priority == y->priority &&
-         Condition::GuaranteedEqual(x->waitp->cond, y->waitp->cond);
-}
-
-// Given the contents of a mutex word containing a PerThreadSynch pointer,
-// return the pointer.
-static inline PerThreadSynch* GetPerThreadSynch(intptr_t v) {
-  return reinterpret_cast<PerThreadSynch*>(v & kMuHigh);
-}
-
-// The next several routines maintain the per-thread next and skip fields
-// used in the Mutex waiter queue.
-// The queue is a circular singly-linked list, of which the "head" is the
-// last element, and head->next if the first element.
-// The skip field has the invariant:
-//   For thread x, x->skip is one of:
-//     - invalid (iff x is not in a Mutex wait queue),
-//     - null, or
-//     - a pointer to a distinct thread waiting later in the same Mutex queue
-//       such that all threads in [x, x->skip] have the same condition, priority
-//       and lock type (MuEquivalentWaiter() is true for all pairs in [x,
-//       x->skip]).
-// In addition, if x->skip is  valid, (x->may_skip || x->skip == null)
-//
-// By the spec of MuEquivalentWaiter(), it is not necessary when removing the
-// first runnable thread y from the front a Mutex queue to adjust the skip
-// field of another thread x because if x->skip==y, x->skip must (have) become
-// invalid before y is removed.  The function TryRemove can remove a specified
-// thread from an arbitrary position in the queue whether runnable or not, so
-// it fixes up skip fields that would otherwise be left dangling.
-// The statement
-//     if (x->may_skip && MuEquivalentWaiter(x, x->next)) { x->skip = x->next; }
-// maintains the invariant provided x is not the last waiter in a Mutex queue
-// The statement
-//          if (x->skip != null) { x->skip = x->skip->skip; }
-// maintains the invariant.
-
-// Returns the last thread y in a mutex waiter queue such that all threads in
-// [x, y] inclusive share the same condition.  Sets skip fields of some threads
-// in that range to optimize future evaluation of Skip() on x values in
-// the range.  Requires thread x is in a mutex waiter queue.
-// The locking is unusual.  Skip() is called under these conditions:
-//   - spinlock is held in call from Enqueue(), with maybe_unlocking == false
-//   - Mutex is held in call from UnlockSlow() by last unlocker, with
-//     maybe_unlocking == true
-//   - both Mutex and spinlock are held in call from DequeueAllWakeable() (from
-//     UnlockSlow()) and TryRemove()
-// These cases are mutually exclusive, so Skip() never runs concurrently
-// with itself on the same Mutex.   The skip chain is used in these other places
-// that cannot occur concurrently:
-//   - FixSkip() (from TryRemove()) - spinlock and Mutex are held)
-//   - Dequeue() (with spinlock and Mutex held)
-//   - UnlockSlow() (with spinlock and Mutex held)
-// A more complex case is Enqueue()
-//   - Enqueue() (with spinlock held and maybe_unlocking == false)
-//               This is the first case in which Skip is called, above.
-//   - Enqueue() (without spinlock held; but queue is empty and being freshly
-//                formed)
-//   - Enqueue() (with spinlock held and maybe_unlocking == true)
-// The first case has mutual exclusion, and the second isolation through
-// working on an otherwise unreachable data structure.
-// In the last case, Enqueue() is required to change no skip/next pointers
-// except those in the added node and the former "head" node.  This implies
-// that the new node is added after head, and so must be the new head or the
-// new front of the queue.
-static PerThreadSynch* Skip(PerThreadSynch* x) {
-  PerThreadSynch* x0 = nullptr;
-  PerThreadSynch* x1 = x;
-  PerThreadSynch* x2 = x->skip;
-  if (x2 != nullptr) {
-    // Each iteration attempts to advance sequence (x0,x1,x2) to next sequence
-    // such that   x1 == x0->skip && x2 == x1->skip
-    while ((x0 = x1, x1 = x2, x2 = x2->skip) != nullptr) {
-      x0->skip = x2;  // short-circuit skip from x0 to x2
-    }
-    x->skip = x1;  // short-circuit skip from x to result
-  }
-  return x1;
-}
-
-// "ancestor" appears before "to_be_removed" in the same Mutex waiter queue.
-// The latter is going to be removed out of order, because of a timeout.
-// Check whether "ancestor" has a skip field pointing to "to_be_removed",
-// and fix it if it does.
-static void FixSkip(PerThreadSynch* ancestor, PerThreadSynch* to_be_removed) {
-  if (ancestor->skip == to_be_removed) {  // ancestor->skip left dangling
-    if (to_be_removed->skip != nullptr) {
-      ancestor->skip = to_be_removed->skip;  // can skip past to_be_removed
-    } else if (ancestor->next != to_be_removed) {  // they are not adjacent
-      ancestor->skip = ancestor->next;             // can skip one past ancestor
-    } else {
-      ancestor->skip = nullptr;  // can't skip at all
-    }
-  }
-}
-
-static void CondVarEnqueue(SynchWaitParams* waitp);
-
-// Enqueue thread "waitp->thread" on a waiter queue.
-// Called with mutex spinlock held if head != nullptr
-// If head==nullptr and waitp->cv_word==nullptr, then Enqueue() is
-// idempotent; it alters no state associated with the existing (empty)
-// queue.
-//
-// If waitp->cv_word == nullptr, queue the thread at either the front or
-// the end (according to its priority) of the circular mutex waiter queue whose
-// head is "head", and return the new head.  mu is the previous mutex state,
-// which contains the reader count (perhaps adjusted for the operation in
-// progress) if the list was empty and a read lock held, and the holder hint if
-// the list was empty and a write lock held.  (flags & kMuIsCond) indicates
-// whether this thread was transferred from a CondVar or is waiting for a
-// non-trivial condition.  In this case, Enqueue() never returns nullptr
-//
-// If waitp->cv_word != nullptr, CondVarEnqueue() is called, and "head" is
-// returned. This mechanism is used by CondVar to queue a thread on the
-// condition variable queue instead of the mutex queue in implementing Wait().
-// In this case, Enqueue() can return nullptr (if head==nullptr).
-static PerThreadSynch* Enqueue(PerThreadSynch* head, SynchWaitParams* waitp,
-                               intptr_t mu, int flags) {
-  // If we have been given a cv_word, call CondVarEnqueue() and return
-  // the previous head of the Mutex waiter queue.
-  if (waitp->cv_word != nullptr) {
-    CondVarEnqueue(waitp);
-    return head;
-  }
-
-  PerThreadSynch* s = waitp->thread;
-  ABSL_RAW_CHECK(
-      s->waitp == nullptr ||    // normal case
-          s->waitp == waitp ||  // Fer()---transfer from condition variable
-          s->suppress_fatal_errors,
-      "detected illegal recursion into Mutex code");
-  s->waitp = waitp;
-  s->skip = nullptr;   // maintain skip invariant (see above)
-  s->may_skip = true;  // always true on entering queue
-  s->wake = false;     // not being woken
-  s->cond_waiter = ((flags & kMuIsCond) != 0);
-#ifdef ABSL_HAVE_PTHREAD_GETSCHEDPARAM
-  if ((flags & kMuIsFer) == 0) {
-    assert(s == Synch_GetPerThread());
-    int64_t now_cycles = CycleClock::Now();
-    if (s->next_priority_read_cycles < now_cycles) {
-      // Every so often, update our idea of the thread's priority.
-      // pthread_getschedparam() is 5% of the block/wakeup time;
-      // CycleClock::Now() is 0.5%.
-      int policy;
-      struct sched_param param;
-      const int err = pthread_getschedparam(pthread_self(), &policy, &param);
-      if (err != 0) {
-        ABSL_RAW_LOG(ERROR, "pthread_getschedparam failed: %d", err);
-      } else {
-        s->priority = param.sched_priority;
-        s->next_priority_read_cycles =
-            now_cycles + static_cast<int64_t>(CycleClock::Frequency());
-      }
-    }
-  }
-#endif
-  if (head == nullptr) {         // s is the only waiter
-    s->next = s;                 // it's the only entry in the cycle
-    s->readers = mu;             // reader count is from mu word
-    s->maybe_unlocking = false;  // no one is searching an empty list
-    head = s;                    // s is new head
-  } else {
-    PerThreadSynch* enqueue_after = nullptr;  // we'll put s after this element
-#ifdef ABSL_HAVE_PTHREAD_GETSCHEDPARAM
-    if (s->priority > head->priority) {  // s's priority is above head's
-      // try to put s in priority-fifo order, or failing that at the front.
-      if (!head->maybe_unlocking) {
-        // No unlocker can be scanning the queue, so we can insert into the
-        // middle of the queue.
-        //
-        // Within a skip chain, all waiters have the same priority, so we can
-        // skip forward through the chains until we find one with a lower
-        // priority than the waiter to be enqueued.
-        PerThreadSynch* advance_to = head;  // next value of enqueue_after
-        do {
-          enqueue_after = advance_to;
-          // (side-effect: optimizes skip chain)
-          advance_to = Skip(enqueue_after->next);
-        } while (s->priority <= advance_to->priority);
-        // termination guaranteed because s->priority > head->priority
-        // and head is the end of a skip chain
-      } else if (waitp->how == kExclusive && waitp->cond == nullptr) {
-        // An unlocker could be scanning the queue, but we know it will recheck
-        // the queue front for writers that have no condition, which is what s
-        // is, so an insert at front is safe.
-        enqueue_after = head;  // add after head, at front
-      }
-    }
-#endif
-    if (enqueue_after != nullptr) {
-      s->next = enqueue_after->next;
-      enqueue_after->next = s;
-
-      // enqueue_after can be: head, Skip(...), or cur.
-      // The first two imply enqueue_after->skip == nullptr, and
-      // the last is used only if MuEquivalentWaiter(s, cur).
-      // We require this because clearing enqueue_after->skip
-      // is impossible; enqueue_after's predecessors might also
-      // incorrectly skip over s if we were to allow other
-      // insertion points.
-      ABSL_RAW_CHECK(enqueue_after->skip == nullptr ||
-                         MuEquivalentWaiter(enqueue_after, s),
-                     "Mutex Enqueue failure");
-
-      if (enqueue_after != head && enqueue_after->may_skip &&
-          MuEquivalentWaiter(enqueue_after, enqueue_after->next)) {
-        // enqueue_after can skip to its new successor, s
-        enqueue_after->skip = enqueue_after->next;
-      }
-      if (MuEquivalentWaiter(s, s->next)) {  // s->may_skip is known to be true
-        s->skip = s->next;                   // s may skip to its successor
-      }
-    } else if ((flags & kMuHasBlocked) &&
-               (s->priority >= head->next->priority) &&
-               (!head->maybe_unlocking ||
-                (waitp->how == kExclusive &&
-                 Condition::GuaranteedEqual(waitp->cond, nullptr)))) {
-      // This thread has already waited, then was woken, then failed to acquire
-      // the mutex and now tries to requeue. Try to requeue it at head,
-      // otherwise it can suffer bad latency (wait whole queue several times).
-      // However, we need to be conservative. First, we need to ensure that we
-      // respect priorities. Then, we need to be careful to not break wait
-      // queue invariants: we require either that unlocker is not scanning
-      // the queue or that the current thread is a writer with no condition
-      // (unlocker will recheck the queue for such waiters).
-      s->next = head->next;
-      head->next = s;
-      if (MuEquivalentWaiter(s, s->next)) {  // s->may_skip is known to be true
-        s->skip = s->next;                   // s may skip to its successor
-      }
-    } else {  // enqueue not done any other way, so
-              // we're inserting s at the back
-      // s will become new head; copy data from head into it
-      s->next = head->next;  // add s after head
-      head->next = s;
-      s->readers = head->readers;  // reader count is from previous head
-      s->maybe_unlocking = head->maybe_unlocking;  // same for unlock hint
-      if (head->may_skip && MuEquivalentWaiter(head, s)) {
-        // head now has successor; may skip
-        head->skip = s;
-      }
-      head = s;  // s is new head
-    }
-  }
-  s->state.store(PerThreadSynch::kQueued, std::memory_order_relaxed);
-  return head;
-}
-
-// Dequeue the successor pw->next of thread pw from the Mutex waiter queue
-// whose last element is head.  The new head element is returned, or null
-// if the list is made empty.
-// Dequeue is called with both spinlock and Mutex held.
-static PerThreadSynch* Dequeue(PerThreadSynch* head, PerThreadSynch* pw) {
-  PerThreadSynch* w = pw->next;
-  pw->next = w->next;                 // snip w out of list
-  if (head == w) {                    // we removed the head
-    head = (pw == w) ? nullptr : pw;  // either emptied list, or pw is new head
-  } else if (pw != head && MuEquivalentWaiter(pw, pw->next)) {
-    // pw can skip to its new successor
-    if (pw->next->skip !=
-        nullptr) {  // either skip to its successors skip target
-      pw->skip = pw->next->skip;
-    } else {  // or to pw's successor
-      pw->skip = pw->next;
-    }
-  }
-  return head;
-}
-
-// Traverse the elements [ pw->next, h] of the circular list whose last element
-// is head.
-// Remove all elements with wake==true and place them in the
-// singly-linked list wake_list in the order found.   Assumes that
-// there is only one such element if the element has how == kExclusive.
-// Return the new head.
-static PerThreadSynch* DequeueAllWakeable(PerThreadSynch* head,
-                                          PerThreadSynch* pw,
-                                          PerThreadSynch** wake_tail) {
-  PerThreadSynch* orig_h = head;
-  PerThreadSynch* w = pw->next;
-  bool skipped = false;
-  do {
-    if (w->wake) {  // remove this element
-      ABSL_RAW_CHECK(pw->skip == nullptr, "bad skip in DequeueAllWakeable");
-      // we're removing pw's successor so either pw->skip is zero or we should
-      // already have removed pw since if pw->skip!=null, pw has the same
-      // condition as w.
-      head = Dequeue(head, pw);
-      w->next = *wake_tail;               // keep list terminated
-      *wake_tail = w;                     // add w to wake_list;
-      wake_tail = &w->next;               // next addition to end
-      if (w->waitp->how == kExclusive) {  // wake at most 1 writer
-        break;
-      }
-    } else {         // not waking this one; skip
-      pw = Skip(w);  // skip as much as possible
-      skipped = true;
-    }
-    w = pw->next;
-    // We want to stop processing after we've considered the original head,
-    // orig_h.  We can't test for w==orig_h in the loop because w may skip over
-    // it; we are guaranteed only that w's predecessor will not skip over
-    // orig_h.  When we've considered orig_h, either we've processed it and
-    // removed it (so orig_h != head), or we considered it and skipped it (so
-    // skipped==true && pw == head because skipping from head always skips by
-    // just one, leaving pw pointing at head).  So we want to
-    // continue the loop with the negation of that expression.
-  } while (orig_h == head && (pw != head || !skipped));
-  return head;
-}
-
-// Try to remove thread s from the list of waiters on this mutex.
-// Does nothing if s is not on the waiter list.
-void Mutex::TryRemove(PerThreadSynch* s) {
-  SchedulingGuard::ScopedDisable disable_rescheduling;
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-  // acquire spinlock & lock
-  if ((v & (kMuWait | kMuSpin | kMuWriter | kMuReader)) == kMuWait &&
-      mu_.compare_exchange_strong(v, v | kMuSpin | kMuWriter,
-                                  std::memory_order_acquire,
-                                  std::memory_order_relaxed)) {
-    PerThreadSynch* h = GetPerThreadSynch(v);
-    if (h != nullptr) {
-      PerThreadSynch* pw = h;  // pw is w's predecessor
-      PerThreadSynch* w;
-      if ((w = pw->next) != s) {  // search for thread,
-        do {                      // processing at least one element
-          // If the current element isn't equivalent to the waiter to be
-          // removed, we can skip the entire chain.
-          if (!MuEquivalentWaiter(s, w)) {
-            pw = Skip(w);  // so skip all that won't match
-            // we don't have to worry about dangling skip fields
-            // in the threads we skipped; none can point to s
-            // because they are in a different equivalence class.
-          } else {          // seeking same condition
-            FixSkip(w, s);  // fix up any skip pointer from w to s
-            pw = w;
-          }
-          // don't search further if we found the thread, or we're about to
-          // process the first thread again.
-        } while ((w = pw->next) != s && pw != h);
-      }
-      if (w == s) {  // found thread; remove it
-        // pw->skip may be non-zero here; the loop above ensured that
-        // no ancestor of s can skip to s, so removal is safe anyway.
-        h = Dequeue(h, pw);
-        s->next = nullptr;
-        s->state.store(PerThreadSynch::kAvailable, std::memory_order_release);
-      }
-    }
-    intptr_t nv;
-    do {  // release spinlock and lock
-      v = mu_.load(std::memory_order_relaxed);
-      nv = v & (kMuDesig | kMuEvent);
-      if (h != nullptr) {
-        nv |= kMuWait | reinterpret_cast<intptr_t>(h);
-        h->readers = 0;              // we hold writer lock
-        h->maybe_unlocking = false;  // finished unlocking
-      }
-    } while (!mu_.compare_exchange_weak(v, nv, std::memory_order_release,
-                                        std::memory_order_relaxed));
-  }
-}
-
-// Wait until thread "s", which must be the current thread, is removed from the
-// this mutex's waiter queue.  If "s->waitp->timeout" has a timeout, wake up
-// if the wait extends past the absolute time specified, even if "s" is still
-// on the mutex queue.  In this case, remove "s" from the queue and return
-// true, otherwise return false.
-void Mutex::Block(PerThreadSynch* s) {
-  while (s->state.load(std::memory_order_acquire) == PerThreadSynch::kQueued) {
-    if (!DecrementSynchSem(this, s, s->waitp->timeout)) {
-      // After a timeout, we go into a spin loop until we remove ourselves
-      // from the queue, or someone else removes us.  We can't be sure to be
-      // able to remove ourselves in a single lock acquisition because this
-      // mutex may be held, and the holder has the right to read the centre
-      // of the waiter queue without holding the spinlock.
-      this->TryRemove(s);
-      int c = 0;
-      while (s->next != nullptr) {
-        c = synchronization_internal::MutexDelay(c, GENTLE);
-        this->TryRemove(s);
-      }
-      if (kDebugMode) {
-        // This ensures that we test the case that TryRemove() is called when s
-        // is not on the queue.
-        this->TryRemove(s);
-      }
-      s->waitp->timeout = KernelTimeout::Never();  // timeout is satisfied
-      s->waitp->cond = nullptr;  // condition no longer relevant for wakeups
-    }
-  }
-  ABSL_RAW_CHECK(s->waitp != nullptr || s->suppress_fatal_errors,
-                 "detected illegal recursion in Mutex code");
-  s->waitp = nullptr;
-}
-
-// Wake thread w, and return the next thread in the list.
-PerThreadSynch* Mutex::Wakeup(PerThreadSynch* w) {
-  PerThreadSynch* next = w->next;
-  w->next = nullptr;
-  w->state.store(PerThreadSynch::kAvailable, std::memory_order_release);
-  IncrementSynchSem(this, w);
-
-  return next;
-}
-
-static GraphId GetGraphIdLocked(Mutex* mu)
-    ABSL_EXCLUSIVE_LOCKS_REQUIRED(deadlock_graph_mu) {
-  if (!deadlock_graph) {  // (re)create the deadlock graph.
-    deadlock_graph =
-        new (base_internal::LowLevelAlloc::Alloc(sizeof(*deadlock_graph)))
-            GraphCycles;
-  }
-  return deadlock_graph->GetId(mu);
-}
-
-static GraphId GetGraphId(Mutex* mu) ABSL_LOCKS_EXCLUDED(deadlock_graph_mu) {
-  base_internal::SpinLockHolder l(deadlock_graph_mu);
-  GraphId id = GetGraphIdLocked(mu);
-  return id;
-}
-
-// Record a lock acquisition.  This is used in debug mode for deadlock
-// detection.  The held_locks pointer points to the relevant data
-// structure for each case.
-static void LockEnter(Mutex* mu, GraphId id, SynchLocksHeld* held_locks) {
-  int n = held_locks->n;
-  int i = 0;
-  while (i != n && held_locks->locks[i].id != id) {
-    i++;
-  }
-  if (i == n) {
-    if (n == ABSL_ARRAYSIZE(held_locks->locks)) {
-      held_locks->overflow = true;  // lost some data
-    } else {                        // we have room for lock
-      held_locks->locks[i].mu = mu;
-      held_locks->locks[i].count = 1;
-      held_locks->locks[i].id = id;
-      held_locks->n = n + 1;
-    }
-  } else {
-    held_locks->locks[i].count++;
-  }
-}
-
-// Record a lock release.  Each call to LockEnter(mu, id, x) should be
-// eventually followed by a call to LockLeave(mu, id, x) by the same thread.
-// It does not process the event if is not needed when deadlock detection is
-// disabled.
-static void LockLeave(Mutex* mu, GraphId id, SynchLocksHeld* held_locks) {
-  int n = held_locks->n;
-  int i = 0;
-  while (i != n && held_locks->locks[i].id != id) {
-    i++;
-  }
-  if (i == n) {
-    if (!held_locks->overflow) {
-      // The deadlock id may have been reassigned after ForgetDeadlockInfo,
-      // but in that case mu should still be present.
-      i = 0;
-      while (i != n && held_locks->locks[i].mu != mu) {
-        i++;
-      }
-      if (i == n) {  // mu missing means releasing unheld lock
-        SynchEvent* mu_events = GetSynchEvent(mu);
-        ABSL_RAW_LOG(FATAL,
-                     "thread releasing lock it does not hold: %p %s; "
-                     ,
-                     static_cast<void*>(mu),
-                     mu_events == nullptr ? "" : mu_events->name);
-      }
-    }
-  } else if (held_locks->locks[i].count == 1) {
-    held_locks->n = n - 1;
-    held_locks->locks[i] = held_locks->locks[n - 1];
-    held_locks->locks[n - 1].id = InvalidGraphId();
-    held_locks->locks[n - 1].mu =
-        nullptr;  // clear mu to please the leak detector.
-  } else {
-    assert(held_locks->locks[i].count > 0);
-    held_locks->locks[i].count--;
-  }
-}
-
-// Call LockEnter() if in debug mode and deadlock detection is enabled.
-static inline void DebugOnlyLockEnter(Mutex* mu) {
-  if (kDebugMode) {
-    if (synch_deadlock_detection.load(std::memory_order_acquire) !=
-        OnDeadlockCycle::kIgnore) {
-      LockEnter(mu, GetGraphId(mu), Synch_GetAllLocks());
-    }
-  }
-}
-
-// Call LockEnter() if in debug mode and deadlock detection is enabled.
-static inline void DebugOnlyLockEnter(Mutex* mu, GraphId id) {
-  if (kDebugMode) {
-    if (synch_deadlock_detection.load(std::memory_order_acquire) !=
-        OnDeadlockCycle::kIgnore) {
-      LockEnter(mu, id, Synch_GetAllLocks());
-    }
-  }
-}
-
-// Call LockLeave() if in debug mode and deadlock detection is enabled.
-static inline void DebugOnlyLockLeave(Mutex* mu) {
-  if (kDebugMode) {
-    if (synch_deadlock_detection.load(std::memory_order_acquire) !=
-        OnDeadlockCycle::kIgnore) {
-      LockLeave(mu, GetGraphId(mu), Synch_GetAllLocks());
-    }
-  }
-}
-
-static char* StackString(void** pcs, int n, char* buf, int maxlen,
-                         bool symbolize) {
-  static constexpr int kSymLen = 200;
-  char sym[kSymLen];
-  int len = 0;
-  for (int i = 0; i != n; i++) {
-    if (len >= maxlen) return buf;
-    size_t count = static_cast<size_t>(maxlen - len);
-    if (symbolize) {
-      if (!absl::Symbolize(pcs[i], sym, kSymLen)) {
-        sym[0] = '\0';
-      }
-      snprintf(buf + len, count, "%s\t@ %p %s\n", (i == 0 ? "\n" : ""), pcs[i],
-               sym);
-    } else {
-      snprintf(buf + len, count, " %p", pcs[i]);
-    }
-    len += static_cast<int>(strlen(&buf[len]));
-  }
-  return buf;
-}
-
-static char* CurrentStackString(char* buf, int maxlen, bool symbolize) {
-  void* pcs[40];
-  return StackString(pcs, absl::GetStackTrace(pcs, ABSL_ARRAYSIZE(pcs), 2), buf,
-                     maxlen, symbolize);
-}
-
-namespace {
-enum {
-  kMaxDeadlockPathLen = 10
-};  // maximum length of a deadlock cycle;
-    // a path this long would be remarkable
-// Buffers required to report a deadlock.
-// We do not allocate them on stack to avoid large stack frame.
-struct DeadlockReportBuffers {
-  char buf[6100];
-  GraphId path[kMaxDeadlockPathLen];
-};
-
-struct ScopedDeadlockReportBuffers {
-  ScopedDeadlockReportBuffers() {
-    b = reinterpret_cast<DeadlockReportBuffers*>(
-        base_internal::LowLevelAlloc::Alloc(sizeof(*b)));
-  }
-  ~ScopedDeadlockReportBuffers() { base_internal::LowLevelAlloc::Free(b); }
-  DeadlockReportBuffers* b;
-};
-
-// Helper to pass to GraphCycles::UpdateStackTrace.
-int GetStack(void** stack, int max_depth) {
-  return absl::GetStackTrace(stack, max_depth, 3);
-}
-}  // anonymous namespace
-
-// Called in debug mode when a thread is about to acquire a lock in a way that
-// may block.
-static GraphId DeadlockCheck(Mutex* mu) {
-  if (synch_deadlock_detection.load(std::memory_order_acquire) ==
-      OnDeadlockCycle::kIgnore) {
-    return InvalidGraphId();
-  }
-
-  SynchLocksHeld* all_locks = Synch_GetAllLocks();
-
-  absl::base_internal::SpinLockHolder lock(deadlock_graph_mu);
-  const GraphId mu_id = GetGraphIdLocked(mu);
-
-  if (all_locks->n == 0) {
-    // There are no other locks held. Return now so that we don't need to
-    // call GetSynchEvent(). This way we do not record the stack trace
-    // for this Mutex. It's ok, since if this Mutex is involved in a deadlock,
-    // it can't always be the first lock acquired by a thread.
-    return mu_id;
-  }
-
-  // We prefer to keep stack traces that show a thread holding and acquiring
-  // as many locks as possible.  This increases the chances that a given edge
-  // in the acquires-before graph will be represented in the stack traces
-  // recorded for the locks.
-  deadlock_graph->UpdateStackTrace(mu_id, all_locks->n + 1, GetStack);
-
-  // For each other mutex already held by this thread:
-  for (int i = 0; i != all_locks->n; i++) {
-    const GraphId other_node_id = all_locks->locks[i].id;
-    const Mutex* other =
-        static_cast<const Mutex*>(deadlock_graph->Ptr(other_node_id));
-    if (other == nullptr) {
-      // Ignore stale lock
-      continue;
-    }
-
-    // Add the acquired-before edge to the graph.
-    if (!deadlock_graph->InsertEdge(other_node_id, mu_id)) {
-      ScopedDeadlockReportBuffers scoped_buffers;
-      DeadlockReportBuffers* b = scoped_buffers.b;
-      static int number_of_reported_deadlocks = 0;
-      number_of_reported_deadlocks++;
-      // Symbolize only 2 first deadlock report to avoid huge slowdowns.
-      bool symbolize = number_of_reported_deadlocks <= 2;
-      ABSL_RAW_LOG(ERROR, "Potential Mutex deadlock: %s",
-                   CurrentStackString(b->buf, sizeof (b->buf), symbolize));
-      size_t len = 0;
-      for (int j = 0; j != all_locks->n; j++) {
-        void* pr = deadlock_graph->Ptr(all_locks->locks[j].id);
-        if (pr != nullptr) {
-          snprintf(b->buf + len, sizeof(b->buf) - len, " %p", pr);
-          len += strlen(&b->buf[len]);
-        }
-      }
-      ABSL_RAW_LOG(ERROR,
-                   "Acquiring absl::Mutex %p while holding %s; a cycle in the "
-                   "historical lock ordering graph has been observed",
-                   static_cast<void*>(mu), b->buf);
-      ABSL_RAW_LOG(ERROR, "Cycle: ");
-      int path_len = deadlock_graph->FindPath(mu_id, other_node_id,
-                                              ABSL_ARRAYSIZE(b->path), b->path);
-      for (int j = 0; j != path_len && j != ABSL_ARRAYSIZE(b->path); j++) {
-        GraphId id = b->path[j];
-        Mutex* path_mu = static_cast<Mutex*>(deadlock_graph->Ptr(id));
-        if (path_mu == nullptr) continue;
-        void** stack;
-        int depth = deadlock_graph->GetStackTrace(id, &stack);
-        snprintf(b->buf, sizeof(b->buf),
-                 "mutex@%p stack: ", static_cast<void*>(path_mu));
-        StackString(stack, depth, b->buf + strlen(b->buf),
-                    static_cast<int>(sizeof(b->buf) - strlen(b->buf)),
-                    symbolize);
-        ABSL_RAW_LOG(ERROR, "%s", b->buf);
-      }
-      if (path_len > static_cast<int>(ABSL_ARRAYSIZE(b->path))) {
-        ABSL_RAW_LOG(ERROR, "(long cycle; list truncated)");
-      }
-      if (synch_deadlock_detection.load(std::memory_order_acquire) ==
-          OnDeadlockCycle::kAbort) {
-        deadlock_graph_mu.unlock();  // avoid deadlock in fatal sighandler
-        ABSL_RAW_LOG(FATAL, "dying due to potential deadlock");
-        return mu_id;
-      }
-      break;  // report at most one potential deadlock per acquisition
-    }
-  }
-
-  return mu_id;
-}
-
-// Invoke DeadlockCheck() iff we're in debug mode and
-// deadlock checking has been enabled.
-static inline GraphId DebugOnlyDeadlockCheck(Mutex* mu) {
-  if (kDebugMode && synch_deadlock_detection.load(std::memory_order_acquire) !=
-                        OnDeadlockCycle::kIgnore) {
-    return DeadlockCheck(mu);
-  } else {
-    return InvalidGraphId();
-  }
-}
-
-void Mutex::ForgetDeadlockInfo() {
-  if (kDebugMode && synch_deadlock_detection.load(std::memory_order_acquire) !=
-                        OnDeadlockCycle::kIgnore) {
-    deadlock_graph_mu.lock();
-    if (deadlock_graph != nullptr) {
-      deadlock_graph->RemoveNode(this);
-    }
-    deadlock_graph_mu.unlock();
-  }
-}
-
-void Mutex::AssertNotHeld() const {
-  // We have the data to allow this check only if in debug mode and deadlock
-  // detection is enabled.
-  if (kDebugMode &&
-      (mu_.load(std::memory_order_relaxed) & (kMuWriter | kMuReader)) != 0 &&
-      synch_deadlock_detection.load(std::memory_order_acquire) !=
-          OnDeadlockCycle::kIgnore) {
-    GraphId id = GetGraphId(const_cast<Mutex*>(this));
-    SynchLocksHeld* locks = Synch_GetAllLocks();
-    for (int i = 0; i != locks->n; i++) {
-      if (locks->locks[i].id == id) {
-        SynchEvent* mu_events = GetSynchEvent(this);
-        ABSL_RAW_LOG(FATAL, "thread should not hold mutex %p %s",
-                     static_cast<const void*>(this),
-                     (mu_events == nullptr ? "" : mu_events->name));
-      }
-    }
-  }
-}
-
-// Attempt to acquire *mu, and return whether successful.  The implementation
-// may spin for a short while if the lock cannot be acquired immediately.
-static bool TryAcquireWithSpinning(std::atomic<intptr_t>* mu) {
-  int c = globals.spinloop_iterations.load(std::memory_order_relaxed);
-  do {  // do/while somewhat faster on AMD
-    intptr_t v = mu->load(std::memory_order_relaxed);
-    if ((v & (kMuReader | kMuEvent)) != 0) {
-      return false;                       // a reader or tracing -> give up
-    } else if (((v & kMuWriter) == 0) &&  // no holder -> try to acquire
-               mu->compare_exchange_strong(v, kMuWriter | v,
-                                           std::memory_order_acquire,
-                                           std::memory_order_relaxed)) {
-      return true;
-    }
-  } while (--c > 0);
-  return false;
-}
-
-void Mutex::lock() {
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, 0);
-  GraphId id = DebugOnlyDeadlockCheck(this);
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-  // try fast acquire, then spin loop
-  if (ABSL_PREDICT_FALSE((v & (kMuWriter | kMuReader | kMuEvent)) != 0) ||
-      ABSL_PREDICT_FALSE(!mu_.compare_exchange_strong(
-          v, kMuWriter | v, std::memory_order_acquire,
-          std::memory_order_relaxed))) {
-    // try spin acquire, then slow loop
-    if (ABSL_PREDICT_FALSE(!TryAcquireWithSpinning(&this->mu_))) {
-      this->LockSlow(kExclusive, nullptr, 0);
-    }
-  }
-  DebugOnlyLockEnter(this, id);
-  ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0);
-}
-
-void Mutex::lock_shared() {
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock);
-  GraphId id = DebugOnlyDeadlockCheck(this);
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-  for (;;) {
-    // If there are non-readers holding the lock, use the slow loop.
-    if (ABSL_PREDICT_FALSE(v & (kMuWriter | kMuWait | kMuEvent)) != 0) {
-      this->LockSlow(kShared, nullptr, 0);
-      break;
-    }
-    // We can avoid the loop and only use the CAS when the lock is free or
-    // only held by readers.
-    if (ABSL_PREDICT_TRUE(mu_.compare_exchange_weak(
-            v, (kMuReader | v) + kMuOne, std::memory_order_acquire,
-            std::memory_order_relaxed))) {
-      break;
-    }
-  }
-  DebugOnlyLockEnter(this, id);
-  ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0);
-}
-
-bool Mutex::LockWhenCommon(const Condition& cond,
-                           synchronization_internal::KernelTimeout t,
-                           bool write) {
-  MuHow how = write ? kExclusive : kShared;
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, TsanFlags(how));
-  GraphId id = DebugOnlyDeadlockCheck(this);
-  bool res = LockSlowWithDeadline(how, &cond, t, 0);
-  DebugOnlyLockEnter(this, id);
-  ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0);
-  return res;
-}
-
-bool Mutex::AwaitCommon(const Condition& cond, KernelTimeout t) {
-  if (kDebugMode) {
-    this->AssertReaderHeld();
-  }
-  if (cond.Eval()) {  // condition already true; nothing to do
-    return true;
-  }
-  MuHow how =
-      (mu_.load(std::memory_order_relaxed) & kMuWriter) ? kExclusive : kShared;
-  ABSL_TSAN_MUTEX_PRE_UNLOCK(this, TsanFlags(how));
-  SynchWaitParams waitp(how, &cond, t, nullptr /*no cvmu*/,
-                        Synch_GetPerThreadAnnotated(this),
-                        nullptr /*no cv_word*/);
-  this->UnlockSlow(&waitp);
-  this->Block(waitp.thread);
-  ABSL_TSAN_MUTEX_POST_UNLOCK(this, TsanFlags(how));
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, TsanFlags(how));
-  this->LockSlowLoop(&waitp, kMuHasBlocked | kMuIsCond);
-  bool res = waitp.cond != nullptr ||  // => cond known true from LockSlowLoop
-             EvalConditionAnnotated(&cond, this, true, false, how == kShared);
-  ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0);
-  ABSL_RAW_CHECK(res || t.has_timeout(),
-                 "condition untrue on return from Await");
-  return res;
-}
-
-bool Mutex::try_lock() {
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_try_lock);
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-  // Try fast acquire.
-  if (ABSL_PREDICT_TRUE((v & (kMuWriter | kMuReader | kMuEvent)) == 0)) {
-    if (ABSL_PREDICT_TRUE(mu_.compare_exchange_strong(
-            v, kMuWriter | v, std::memory_order_acquire,
-            std::memory_order_relaxed))) {
-      DebugOnlyLockEnter(this);
-      ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_try_lock, 0);
-      return true;
-    }
-  } else if (ABSL_PREDICT_FALSE((v & kMuEvent) != 0)) {
-    // We're recording events.
-    return TryLockSlow();
-  }
-  ABSL_TSAN_MUTEX_POST_LOCK(
-      this, __tsan_mutex_try_lock | __tsan_mutex_try_lock_failed, 0);
-  return false;
-}
-
-ABSL_ATTRIBUTE_NOINLINE bool Mutex::TryLockSlow() {
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-  if ((v & kExclusive->slow_need_zero) == 0 &&  // try fast acquire
-      mu_.compare_exchange_strong(
-          v, (kExclusive->fast_or | v) + kExclusive->fast_add,
-          std::memory_order_acquire, std::memory_order_relaxed)) {
-    DebugOnlyLockEnter(this);
-    PostSynchEvent(this, SYNCH_EV_TRYLOCK_SUCCESS);
-    ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_try_lock, 0);
-    return true;
-  }
-  PostSynchEvent(this, SYNCH_EV_TRYLOCK_FAILED);
-  ABSL_TSAN_MUTEX_POST_LOCK(
-      this, __tsan_mutex_try_lock | __tsan_mutex_try_lock_failed, 0);
-  return false;
-}
-
-bool Mutex::try_lock_shared() {
-  ABSL_TSAN_MUTEX_PRE_LOCK(this,
-                           __tsan_mutex_read_lock | __tsan_mutex_try_lock);
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-  // Clang tends to unroll the loop when compiling with optimization.
-  // But in this case it just unnecessary increases code size.
-  // If CAS is failing due to contention, the jump cost is negligible.
-#if defined(__clang__)
-#pragma nounroll
-#endif
-  // The while-loops (here and below) iterate only if the mutex word keeps
-  // changing (typically because the reader count changes) under the CAS.
-  // We limit the number of attempts to avoid having to think about livelock.
-  for (int loop_limit = 5; loop_limit != 0; loop_limit--) {
-    if (ABSL_PREDICT_FALSE((v & (kMuWriter | kMuWait | kMuEvent)) != 0)) {
-      break;
-    }
-    if (ABSL_PREDICT_TRUE(mu_.compare_exchange_strong(
-            v, (kMuReader | v) + kMuOne, std::memory_order_acquire,
-            std::memory_order_relaxed))) {
-      DebugOnlyLockEnter(this);
-      ABSL_TSAN_MUTEX_POST_LOCK(
-          this, __tsan_mutex_read_lock | __tsan_mutex_try_lock, 0);
-      return true;
-    }
-  }
-  if (ABSL_PREDICT_TRUE((v & kMuEvent) == 0)) {
-    ABSL_TSAN_MUTEX_POST_LOCK(this,
-                              __tsan_mutex_read_lock | __tsan_mutex_try_lock |
-                                  __tsan_mutex_try_lock_failed,
-                              0);
-    return false;
-  }
-  // we're recording events
-  return ReaderTryLockSlow();
-}
-
-ABSL_ATTRIBUTE_NOINLINE bool Mutex::ReaderTryLockSlow() {
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-#if defined(__clang__)
-#pragma nounroll
-#endif
-  for (int loop_limit = 5; loop_limit != 0; loop_limit--) {
-    if ((v & kShared->slow_need_zero) == 0 &&
-        mu_.compare_exchange_strong(v, (kMuReader | v) + kMuOne,
-                                    std::memory_order_acquire,
-                                    std::memory_order_relaxed)) {
-      DebugOnlyLockEnter(this);
-      PostSynchEvent(this, SYNCH_EV_READERTRYLOCK_SUCCESS);
-      ABSL_TSAN_MUTEX_POST_LOCK(
-          this, __tsan_mutex_read_lock | __tsan_mutex_try_lock, 0);
-      return true;
-    }
-  }
-  PostSynchEvent(this, SYNCH_EV_READERTRYLOCK_FAILED);
-  ABSL_TSAN_MUTEX_POST_LOCK(this,
-                            __tsan_mutex_read_lock | __tsan_mutex_try_lock |
-                                __tsan_mutex_try_lock_failed,
-                            0);
-  return false;
-}
-
-void Mutex::unlock() {
-  ABSL_TSAN_MUTEX_PRE_UNLOCK(this, 0);
-  DebugOnlyLockLeave(this);
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-
-  if (kDebugMode && ((v & (kMuWriter | kMuReader)) != kMuWriter)) {
-    ABSL_RAW_LOG(FATAL, "Mutex unlocked when destroyed or not locked: v=0x%x",
-                 static_cast<unsigned>(v));
-  }
-
-  // should_try_cas is whether we'll try a compare-and-swap immediately.
-  // NOTE: optimized out when kDebugMode is false.
-  bool should_try_cas = ((v & (kMuEvent | kMuWriter)) == kMuWriter &&
-                         (v & (kMuWait | kMuDesig)) != kMuWait);
-
-  // But, we can use an alternate computation of it, that compilers
-  // currently don't find on their own.  When that changes, this function
-  // can be simplified.
-  //
-  // should_try_cas is true iff the bits satisfy the following conditions:
-  //
-  //                   Ev Wr Wa De
-  // equal to           0  1
-  // and not equal to         1  0
-  //
-  // after xoring by    0  1  0  1,  this is equivalent to:
-  //
-  // equal to           0  0
-  // and not equal to         1  1,  which is the same as:
-  //
-  // smaller than       0  0  1  1
-  static_assert(kMuEvent > kMuWait, "Needed for should_try_cas_fast");
-  static_assert(kMuEvent > kMuDesig, "Needed for should_try_cas_fast");
-  static_assert(kMuWriter > kMuWait, "Needed for should_try_cas_fast");
-  static_assert(kMuWriter > kMuDesig, "Needed for should_try_cas_fast");
-
-  bool should_try_cas_fast =
-      ((v ^ (kMuWriter | kMuDesig)) &
-       (kMuEvent | kMuWriter | kMuWait | kMuDesig)) < (kMuWait | kMuDesig);
-
-  if (kDebugMode && should_try_cas != should_try_cas_fast) {
-    // We would usually use PRIdPTR here, but is not correctly implemented
-    // within the android toolchain.
-    ABSL_RAW_LOG(FATAL, "internal logic error %llx %llx %llx\n",
-                 static_cast<long long>(v),
-                 static_cast<long long>(should_try_cas),
-                 static_cast<long long>(should_try_cas_fast));
-  }
-  if (should_try_cas_fast &&
-      mu_.compare_exchange_strong(v, v & ~(kMuWrWait | kMuWriter),
-                                  std::memory_order_release,
-                                  std::memory_order_relaxed)) {
-    // fast writer release (writer with no waiters or with designated waker)
-  } else {
-    this->UnlockSlow(nullptr /*no waitp*/);  // take slow path
-  }
-  ABSL_TSAN_MUTEX_POST_UNLOCK(this, 0);
-}
-
-// Requires v to represent a reader-locked state.
-static bool ExactlyOneReader(intptr_t v) {
-  assert((v & (kMuWriter | kMuReader)) == kMuReader);
-  assert((v & kMuHigh) != 0);
-  // The more straightforward "(v & kMuHigh) == kMuOne" also works, but
-  // on some architectures the following generates slightly smaller code.
-  // It may be faster too.
-  constexpr intptr_t kMuMultipleWaitersMask = kMuHigh ^ kMuOne;
-  return (v & kMuMultipleWaitersMask) == 0;
-}
-
-void Mutex::unlock_shared() {
-  ABSL_TSAN_MUTEX_PRE_UNLOCK(this, __tsan_mutex_read_lock);
-  DebugOnlyLockLeave(this);
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-  assert((v & (kMuWriter | kMuReader)) == kMuReader);
-  for (;;) {
-    if (ABSL_PREDICT_FALSE((v & (kMuReader | kMuWait | kMuEvent)) !=
-                           kMuReader)) {
-      this->UnlockSlow(nullptr /*no waitp*/);  // take slow path
-      break;
-    }
-    // fast reader release (reader with no waiters)
-    intptr_t clear = ExactlyOneReader(v) ? kMuReader | kMuOne : kMuOne;
-    if (ABSL_PREDICT_TRUE(
-            mu_.compare_exchange_strong(v, v - clear, std::memory_order_release,
-                                        std::memory_order_relaxed))) {
-      break;
-    }
-  }
-  ABSL_TSAN_MUTEX_POST_UNLOCK(this, __tsan_mutex_read_lock);
-}
-
-// Clears the designated waker flag in the mutex if this thread has blocked, and
-// therefore may be the designated waker.
-static intptr_t ClearDesignatedWakerMask(int flag) {
-  assert(flag >= 0);
-  assert(flag <= 1);
-  switch (flag) {
-    case 0:  // not blocked
-      return ~static_cast<intptr_t>(0);
-    case 1:  // blocked; turn off the designated waker bit
-      return ~static_cast<intptr_t>(kMuDesig);
-  }
-  ABSL_UNREACHABLE();
-}
-
-// Conditionally ignores the existence of waiting writers if a reader that has
-// already blocked once wakes up.
-static intptr_t IgnoreWaitingWritersMask(int flag) {
-  assert(flag >= 0);
-  assert(flag <= 1);
-  switch (flag) {
-    case 0:  // not blocked
-      return ~static_cast<intptr_t>(0);
-    case 1:  // blocked; pretend there are no waiting writers
-      return ~static_cast<intptr_t>(kMuWrWait);
-  }
-  ABSL_UNREACHABLE();
-}
-
-// Internal version of LockWhen().  See LockSlowWithDeadline()
-ABSL_ATTRIBUTE_NOINLINE void Mutex::LockSlow(MuHow how, const Condition* cond,
-                                             int flags) {
-  // Note: we specifically initialize spinloop_iterations after the first use
-  // in TryAcquireWithSpinning so that Lock function does not have any non-tail
-  // calls and consequently a stack frame. It's fine to have spinloop_iterations
-  // uninitialized (meaning no spinning) in all initial uncontended Lock calls
-  // and in the first contended call. After that we will have
-  // spinloop_iterations properly initialized.
-  if (ABSL_PREDICT_FALSE(
-          globals.spinloop_iterations.load(std::memory_order_relaxed) == 0)) {
-    if (absl::base_internal::NumCPUs() > 1) {
-      // If this is multiprocessor, allow spinning.
-      globals.spinloop_iterations.store(1500, std::memory_order_relaxed);
-    } else {
-      // If this a uniprocessor, only yield/sleep.
-      globals.spinloop_iterations.store(-1, std::memory_order_relaxed);
-    }
-  }
-  ABSL_RAW_CHECK(
-      this->LockSlowWithDeadline(how, cond, KernelTimeout::Never(), flags),
-      "condition untrue on return from LockSlow");
-}
-
-// Compute cond->Eval() and tell race detectors that we do it under mutex mu.
-static inline bool EvalConditionAnnotated(const Condition* cond, Mutex* mu,
-                                          bool locking, bool trylock,
-                                          bool read_lock) {
-  // Delicate annotation dance.
-  // We are currently inside of read/write lock/unlock operation.
-  // All memory accesses are ignored inside of mutex operations + for unlock
-  // operation tsan considers that we've already released the mutex.
-  bool res = false;
-#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE
-  const uint32_t flags = read_lock ? __tsan_mutex_read_lock : 0;
-  const uint32_t tryflags = flags | (trylock ? __tsan_mutex_try_lock : 0);
-#endif
-  if (locking) {
-    // For lock we pretend that we have finished the operation,
-    // evaluate the predicate, then unlock the mutex and start locking it again
-    // to match the annotation at the end of outer lock operation.
-    // Note: we can't simply do POST_LOCK, Eval, PRE_LOCK, because then tsan
-    // will think the lock acquisition is recursive which will trigger
-    // deadlock detector.
-    ABSL_TSAN_MUTEX_POST_LOCK(mu, tryflags, 0);
-    res = cond->Eval();
-    // There is no "try" version of Unlock, so use flags instead of tryflags.
-    ABSL_TSAN_MUTEX_PRE_UNLOCK(mu, flags);
-    ABSL_TSAN_MUTEX_POST_UNLOCK(mu, flags);
-    ABSL_TSAN_MUTEX_PRE_LOCK(mu, tryflags);
-  } else {
-    // Similarly, for unlock we pretend that we have unlocked the mutex,
-    // lock the mutex, evaluate the predicate, and start unlocking it again
-    // to match the annotation at the end of outer unlock operation.
-    ABSL_TSAN_MUTEX_POST_UNLOCK(mu, flags);
-    ABSL_TSAN_MUTEX_PRE_LOCK(mu, flags);
-    ABSL_TSAN_MUTEX_POST_LOCK(mu, flags, 0);
-    res = cond->Eval();
-    ABSL_TSAN_MUTEX_PRE_UNLOCK(mu, flags);
-  }
-  // Prevent unused param warnings in non-TSAN builds.
-  static_cast<void>(mu);
-  static_cast<void>(trylock);
-  static_cast<void>(read_lock);
-  return res;
-}
-
-// Compute cond->Eval() hiding it from race detectors.
-// We are hiding it because inside of UnlockSlow we can evaluate a predicate
-// that was just added by a concurrent Lock operation; Lock adds the predicate
-// to the internal Mutex list without actually acquiring the Mutex
-// (it only acquires the internal spinlock, which is rightfully invisible for
-// tsan). As the result there is no tsan-visible synchronization between the
-// addition and this thread. So if we would enable race detection here,
-// it would race with the predicate initialization.
-static inline bool EvalConditionIgnored(Mutex* mu, const Condition* cond) {
-  // Memory accesses are already ignored inside of lock/unlock operations,
-  // but synchronization operations are also ignored. When we evaluate the
-  // predicate we must ignore only memory accesses but not synchronization,
-  // because missed synchronization can lead to false reports later.
-  // So we "divert" (which un-ignores both memory accesses and synchronization)
-  // and then separately turn on ignores of memory accesses.
-  ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0);
-  ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN();
-  bool res = cond->Eval();
-  ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_END();
-  ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0);
-  static_cast<void>(mu);  // Prevent unused param warning in non-TSAN builds.
-  return res;
-}
-
-// Internal equivalent of *LockWhenWithDeadline(), where
-//   "t" represents the absolute timeout; !t.has_timeout() means "forever".
-//   "how" is "kShared" (for ReaderLockWhen) or "kExclusive" (for LockWhen)
-// In flags, bits are ored together:
-// - kMuHasBlocked indicates that the client has already blocked on the call so
-//   the designated waker bit must be cleared and waiting writers should not
-//   obstruct this call
-// - kMuIsCond indicates that this is a conditional acquire (condition variable,
-//   Await,  LockWhen) so contention profiling should be suppressed.
-bool Mutex::LockSlowWithDeadline(MuHow how, const Condition* cond,
-                                 KernelTimeout t, int flags) {
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-  bool unlock = false;
-  if ((v & how->fast_need_zero) == 0 &&  // try fast acquire
-      mu_.compare_exchange_strong(
-          v,
-          (how->fast_or |
-           (v & ClearDesignatedWakerMask(flags & kMuHasBlocked))) +
-              how->fast_add,
-          std::memory_order_acquire, std::memory_order_relaxed)) {
-    if (cond == nullptr ||
-        EvalConditionAnnotated(cond, this, true, false, how == kShared)) {
-      return true;
-    }
-    unlock = true;
-  }
-  SynchWaitParams waitp(how, cond, t, nullptr /*no cvmu*/,
-                        Synch_GetPerThreadAnnotated(this),
-                        nullptr /*no cv_word*/);
-  if (cond != nullptr) {
-    flags |= kMuIsCond;
-  }
-  if (unlock) {
-    this->UnlockSlow(&waitp);
-    this->Block(waitp.thread);
-    flags |= kMuHasBlocked;
-  }
-  this->LockSlowLoop(&waitp, flags);
-  return waitp.cond != nullptr ||  // => cond known true from LockSlowLoop
-         cond == nullptr ||
-         EvalConditionAnnotated(cond, this, true, false, how == kShared);
-}
-
-// RAW_CHECK_FMT() takes a condition, a printf-style format string, and
-// the printf-style argument list.   The format string must be a literal.
-// Arguments after the first are not evaluated unless the condition is true.
-#define RAW_CHECK_FMT(cond, ...)                                   \
-  do {                                                             \
-    if (ABSL_PREDICT_FALSE(!(cond))) {                             \
-      ABSL_RAW_LOG(FATAL, "Check " #cond " failed: " __VA_ARGS__); \
-    }                                                              \
-  } while (0)
-
-static void CheckForMutexCorruption(intptr_t v, const char* label) {
-  // Test for either of two situations that should not occur in v:
-  //   kMuWriter and kMuReader
-  //   kMuWrWait and !kMuWait
-  const uintptr_t w = static_cast<uintptr_t>(v ^ kMuWait);
-  // By flipping that bit, we can now test for:
-  //   kMuWriter and kMuReader in w
-  //   kMuWrWait and kMuWait in w
-  // We've chosen these two pairs of values to be so that they will overlap,
-  // respectively, when the word is left shifted by three.  This allows us to
-  // save a branch in the common (correct) case of them not being coincident.
-  static_assert(kMuReader << 3 == kMuWriter, "must match");
-  static_assert(kMuWait << 3 == kMuWrWait, "must match");
-  if (ABSL_PREDICT_TRUE((w & (w << 3) & (kMuWriter | kMuWrWait)) == 0)) return;
-  RAW_CHECK_FMT((v & (kMuWriter | kMuReader)) != (kMuWriter | kMuReader),
-                "%s: Mutex corrupt: both reader and writer lock held: %p",
-                label, reinterpret_cast<void*>(v));
-  RAW_CHECK_FMT((v & (kMuWait | kMuWrWait)) != kMuWrWait,
-                "%s: Mutex corrupt: waiting writer with no waiters: %p", label,
-                reinterpret_cast<void*>(v));
-  assert(false);
-}
-
-void Mutex::LockSlowLoop(SynchWaitParams* waitp, int flags) {
-  SchedulingGuard::ScopedDisable disable_rescheduling;
-  int c = 0;
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-  if ((v & kMuEvent) != 0) {
-    PostSynchEvent(
-        this, waitp->how == kExclusive ? SYNCH_EV_LOCK : SYNCH_EV_READERLOCK);
-  }
-  ABSL_RAW_CHECK(
-      waitp->thread->waitp == nullptr || waitp->thread->suppress_fatal_errors,
-      "detected illegal recursion into Mutex code");
-  for (;;) {
-    v = mu_.load(std::memory_order_relaxed);
-    CheckForMutexCorruption(v, "Lock");
-    if ((v & waitp->how->slow_need_zero) == 0) {
-      if (mu_.compare_exchange_strong(
-              v,
-              (waitp->how->fast_or |
-               (v & ClearDesignatedWakerMask(flags & kMuHasBlocked))) +
-                  waitp->how->fast_add,
-              std::memory_order_acquire, std::memory_order_relaxed)) {
-        if (waitp->cond == nullptr ||
-            EvalConditionAnnotated(waitp->cond, this, true, false,
-                                   waitp->how == kShared)) {
-          break;  // we timed out, or condition true, so return
-        }
-        this->UnlockSlow(waitp);  // got lock but condition false
-        this->Block(waitp->thread);
-        flags |= kMuHasBlocked;
-        c = 0;
-      }
-    } else {  // need to access waiter list
-      bool dowait = false;
-      if ((v & (kMuSpin | kMuWait)) == 0) {  // no waiters
-        // This thread tries to become the one and only waiter.
-        PerThreadSynch* new_h = Enqueue(nullptr, waitp, v, flags);
-        intptr_t nv =
-            (v & ClearDesignatedWakerMask(flags & kMuHasBlocked) & kMuLow) |
-            kMuWait;
-        ABSL_RAW_CHECK(new_h != nullptr, "Enqueue to empty list failed");
-        if (waitp->how == kExclusive && (v & kMuReader) != 0) {
-          nv |= kMuWrWait;
-        }
-        if (mu_.compare_exchange_strong(
-                v, reinterpret_cast<intptr_t>(new_h) | nv,
-                std::memory_order_release, std::memory_order_relaxed)) {
-          dowait = true;
-        } else {  // attempted Enqueue() failed
-          // zero out the waitp field set by Enqueue()
-          waitp->thread->waitp = nullptr;
-        }
-      } else if ((v & waitp->how->slow_inc_need_zero &
-                  IgnoreWaitingWritersMask(flags & kMuHasBlocked)) == 0) {
-        // This is a reader that needs to increment the reader count,
-        // but the count is currently held in the last waiter.
-        if (mu_.compare_exchange_strong(
-                v,
-                (v & ClearDesignatedWakerMask(flags & kMuHasBlocked)) |
-                    kMuSpin | kMuReader,
-                std::memory_order_acquire, std::memory_order_relaxed)) {
-          PerThreadSynch* h = GetPerThreadSynch(v);
-          h->readers += kMuOne;  // inc reader count in waiter
-          do {                   // release spinlock
-            v = mu_.load(std::memory_order_relaxed);
-          } while (!mu_.compare_exchange_weak(v, (v & ~kMuSpin) | kMuReader,
-                                              std::memory_order_release,
-                                              std::memory_order_relaxed));
-          if (waitp->cond == nullptr ||
-              EvalConditionAnnotated(waitp->cond, this, true, false,
-                                     waitp->how == kShared)) {
-            break;  // we timed out, or condition true, so return
-          }
-          this->UnlockSlow(waitp);  // got lock but condition false
-          this->Block(waitp->thread);
-          flags |= kMuHasBlocked;
-          c = 0;
-        }
-      } else if ((v & kMuSpin) == 0 &&  // attempt to queue ourselves
-                 mu_.compare_exchange_strong(
-                     v,
-                     (v & ClearDesignatedWakerMask(flags & kMuHasBlocked)) |
-                         kMuSpin | kMuWait,
-                     std::memory_order_acquire, std::memory_order_relaxed)) {
-        PerThreadSynch* h = GetPerThreadSynch(v);
-        PerThreadSynch* new_h = Enqueue(h, waitp, v, flags);
-        intptr_t wr_wait = 0;
-        ABSL_RAW_CHECK(new_h != nullptr, "Enqueue to list failed");
-        if (waitp->how == kExclusive && (v & kMuReader) != 0) {
-          wr_wait = kMuWrWait;  // give priority to a waiting writer
-        }
-        do {  // release spinlock
-          v = mu_.load(std::memory_order_relaxed);
-        } while (!mu_.compare_exchange_weak(
-            v,
-            (v & (kMuLow & ~kMuSpin)) | kMuWait | wr_wait |
-                reinterpret_cast<intptr_t>(new_h),
-            std::memory_order_release, std::memory_order_relaxed));
-        dowait = true;
-      }
-      if (dowait) {
-        this->Block(waitp->thread);  // wait until removed from list or timeout
-        flags |= kMuHasBlocked;
-        c = 0;
-      }
-    }
-    ABSL_RAW_CHECK(
-        waitp->thread->waitp == nullptr || waitp->thread->suppress_fatal_errors,
-        "detected illegal recursion into Mutex code");
-    // delay, then try again
-    c = synchronization_internal::MutexDelay(c, GENTLE);
-  }
-  ABSL_RAW_CHECK(
-      waitp->thread->waitp == nullptr || waitp->thread->suppress_fatal_errors,
-      "detected illegal recursion into Mutex code");
-  if ((v & kMuEvent) != 0) {
-    PostSynchEvent(this, waitp->how == kExclusive
-                             ? SYNCH_EV_LOCK_RETURNING
-                             : SYNCH_EV_READERLOCK_RETURNING);
-  }
-}
-
-// Unlock this mutex, which is held by the current thread.
-// If waitp is non-zero, it must be the wait parameters for the current thread
-// which holds the lock but is not runnable because its condition is false
-// or it is in the process of blocking on a condition variable; it must requeue
-// itself on the mutex/condvar to wait for its condition to become true.
-ABSL_ATTRIBUTE_NOINLINE void Mutex::UnlockSlow(SynchWaitParams* waitp) {
-  SchedulingGuard::ScopedDisable disable_rescheduling;
-  intptr_t v = mu_.load(std::memory_order_relaxed);
-  this->AssertReaderHeld();
-  CheckForMutexCorruption(v, "Unlock");
-  if ((v & kMuEvent) != 0) {
-    PostSynchEvent(
-        this, (v & kMuWriter) != 0 ? SYNCH_EV_UNLOCK : SYNCH_EV_READERUNLOCK);
-  }
-  int c = 0;
-  // the waiter under consideration to wake, or zero
-  PerThreadSynch* w = nullptr;
-  // the predecessor to w or zero
-  PerThreadSynch* pw = nullptr;
-  // head of the list searched previously, or zero
-  PerThreadSynch* old_h = nullptr;
-  // a condition that's known to be false.
-  PerThreadSynch* wake_list = kPerThreadSynchNull;  // list of threads to wake
-  intptr_t wr_wait = 0;  // set to kMuWrWait if we wake a reader and a
-                         // later writer could have acquired the lock
-                         // (starvation avoidance)
-  ABSL_RAW_CHECK(waitp == nullptr || waitp->thread->waitp == nullptr ||
-                     waitp->thread->suppress_fatal_errors,
-                 "detected illegal recursion into Mutex code");
-  // This loop finds threads wake_list to wakeup if any, and removes them from
-  // the list of waiters.  In addition, it places waitp.thread on the queue of
-  // waiters if waitp is non-zero.
-  for (;;) {
-    v = mu_.load(std::memory_order_relaxed);
-    if ((v & kMuWriter) != 0 && (v & (kMuWait | kMuDesig)) != kMuWait &&
-        waitp == nullptr) {
-      // fast writer release (writer with no waiters or with designated waker)
-      if (mu_.compare_exchange_strong(v, v & ~(kMuWrWait | kMuWriter),
-                                      std::memory_order_release,
-                                      std::memory_order_relaxed)) {
-        return;
-      }
-    } else if ((v & (kMuReader | kMuWait)) == kMuReader && waitp == nullptr) {
-      // fast reader release (reader with no waiters)
-      intptr_t clear = ExactlyOneReader(v) ? kMuReader | kMuOne : kMuOne;
-      if (mu_.compare_exchange_strong(v, v - clear, std::memory_order_release,
-                                      std::memory_order_relaxed)) {
-        return;
-      }
-    } else if ((v & kMuSpin) == 0 &&  // attempt to get spinlock
-               mu_.compare_exchange_strong(v, v | kMuSpin,
-                                           std::memory_order_acquire,
-                                           std::memory_order_relaxed)) {
-      if ((v & kMuWait) == 0) {  // no one to wake
-        intptr_t nv;
-        bool do_enqueue = true;  // always Enqueue() the first time
-        ABSL_RAW_CHECK(waitp != nullptr,
-                       "UnlockSlow is confused");  // about to sleep
-        do {  // must loop to release spinlock as reader count may change
-          v = mu_.load(std::memory_order_relaxed);
-          // decrement reader count if there are readers
-          intptr_t new_readers = (v >= kMuOne) ? v - kMuOne : v;
-          PerThreadSynch* new_h = nullptr;
-          if (do_enqueue) {
-            // If we are enqueuing on a CondVar (waitp->cv_word != nullptr) then
-            // we must not retry here.  The initial attempt will always have
-            // succeeded, further attempts would enqueue us against *this due to
-            // Fer() handling.
-            do_enqueue = (waitp->cv_word == nullptr);
-            new_h = Enqueue(nullptr, waitp, new_readers, kMuIsCond);
-          }
-          intptr_t clear = kMuWrWait | kMuWriter;  // by default clear write bit
-          if ((v & kMuWriter) == 0 && ExactlyOneReader(v)) {  // last reader
-            clear = kMuWrWait | kMuReader;                    // clear read bit
-          }
-          nv = (v & kMuLow & ~clear & ~kMuSpin);
-          if (new_h != nullptr) {
-            nv |= kMuWait | reinterpret_cast<intptr_t>(new_h);
-          } else {  // new_h could be nullptr if we queued ourselves on a
-                    // CondVar
-            // In that case, we must place the reader count back in the mutex
-            // word, as Enqueue() did not store it in the new waiter.
-            nv |= new_readers & kMuHigh;
-          }
-          // release spinlock & our lock; retry if reader-count changed
-          // (writer count cannot change since we hold lock)
-        } while (!mu_.compare_exchange_weak(v, nv, std::memory_order_release,
-                                            std::memory_order_relaxed));
-        break;
-      }
-
-      // There are waiters.
-      // Set h to the head of the circular waiter list.
-      PerThreadSynch* h = GetPerThreadSynch(v);
-      if ((v & kMuReader) != 0 && (h->readers & kMuHigh) > kMuOne) {
-        // a reader but not the last
-        h->readers -= kMuOne;    // release our lock
-        intptr_t nv = v;         // normally just release spinlock
-        if (waitp != nullptr) {  // but waitp!=nullptr => must queue ourselves
-          PerThreadSynch* new_h = Enqueue(h, waitp, v, kMuIsCond);
-          ABSL_RAW_CHECK(new_h != nullptr,
-                         "waiters disappeared during Enqueue()!");
-          nv &= kMuLow;
-          nv |= kMuWait | reinterpret_cast<intptr_t>(new_h);
-        }
-        mu_.store(nv, std::memory_order_release);  // release spinlock
-        // can release with a store because there were waiters
-        break;
-      }
-
-      // Either we didn't search before, or we marked the queue
-      // as "maybe_unlocking" and no one else should have changed it.
-      ABSL_RAW_CHECK(old_h == nullptr || h->maybe_unlocking,
-                     "Mutex queue changed beneath us");
-
-      // The lock is becoming free, and there's a waiter
-      if (old_h != nullptr &&
-          !old_h->may_skip) {    // we used old_h as a terminator
-        old_h->may_skip = true;  // allow old_h to skip once more
-        ABSL_RAW_CHECK(old_h->skip == nullptr, "illegal skip from head");
-        if (h != old_h && MuEquivalentWaiter(old_h, old_h->next)) {
-          old_h->skip = old_h->next;  // old_h not head & can skip to successor
-        }
-      }
-      if (h->next->waitp->how == kExclusive &&
-          h->next->waitp->cond == nullptr) {
-        // easy case: writer with no condition; no need to search
-        pw = h;  // wake w, the successor of h (=pw)
-        w = h->next;
-        w->wake = true;
-        // We are waking up a writer.  This writer may be racing against
-        // an already awake reader for the lock.  We want the
-        // writer to usually win this race,
-        // because if it doesn't, we can potentially keep taking a reader
-        // perpetually and writers will starve.  Worse than
-        // that, this can also starve other readers if kMuWrWait gets set
-        // later.
-        wr_wait = kMuWrWait;
-      } else if (w != nullptr && (w->waitp->how == kExclusive || h == old_h)) {
-        // we found a waiter w to wake on a previous iteration and either it's
-        // a writer, or we've searched the entire list so we have all the
-        // readers.
-        if (pw == nullptr) {  // if w's predecessor is unknown, it must be h
-          pw = h;
-        }
-      } else {
-        // At this point we don't know all the waiters to wake, and the first
-        // waiter has a condition or is a reader.  We avoid searching over
-        // waiters we've searched on previous iterations by starting at
-        // old_h if it's set.  If old_h==h, there's no one to wakeup at all.
-        if (old_h == h) {  // we've searched before, and nothing's new
-                           // so there's no one to wake.
-          intptr_t nv = (v & ~(kMuReader | kMuWriter | kMuWrWait));
-          h->readers = 0;
-          h->maybe_unlocking = false;  // finished unlocking
-          if (waitp != nullptr) {      // we must queue ourselves and sleep
-            PerThreadSynch* new_h = Enqueue(h, waitp, v, kMuIsCond);
-            nv &= kMuLow;
-            if (new_h != nullptr) {
-              nv |= kMuWait | reinterpret_cast<intptr_t>(new_h);
-            }  // else new_h could be nullptr if we queued ourselves on a
-               // CondVar
-          }
-          // release spinlock & lock
-          // can release with a store because there were waiters
-          mu_.store(nv, std::memory_order_release);
-          break;
-        }
-
-        // set up to walk the list
-        PerThreadSynch* w_walk;   // current waiter during list walk
-        PerThreadSynch* pw_walk;  // previous waiter during list walk
-        if (old_h != nullptr) {   // we've searched up to old_h before
-          pw_walk = old_h;
-          w_walk = old_h->next;
-        } else {  // no prior search, start at beginning
-          pw_walk =
-              nullptr;  // h->next's predecessor may change; don't record it
-          w_walk = h->next;
-        }
-
-        h->may_skip = false;  // ensure we never skip past h in future searches
-                              // even if other waiters are queued after it.
-        ABSL_RAW_CHECK(h->skip == nullptr, "illegal skip from head");
-
-        h->maybe_unlocking = true;  // we're about to scan the waiter list
-                                    // without the spinlock held.
-                                    // Enqueue must be conservative about
-                                    // priority queuing.
-
-        // We must release the spinlock to evaluate the conditions.
-        mu_.store(v, std::memory_order_release);  // release just spinlock
-        // can release with a store because there were waiters
-
-        // h is the last waiter queued, and w_walk the first unsearched waiter.
-        // Without the spinlock, the locations mu_ and h->next may now change
-        // underneath us, but since we hold the lock itself, the only legal
-        // change is to add waiters between h and w_walk.  Therefore, it's safe
-        // to walk the path from w_walk to h inclusive. (TryRemove() can remove
-        // a waiter anywhere, but it acquires both the spinlock and the Mutex)
-
-        old_h = h;  // remember we searched to here
-
-        // Walk the path upto and including h looking for waiters we can wake.
-        while (pw_walk != h) {
-          w_walk->wake = false;
-          if (w_walk->waitp->cond ==
-                  nullptr ||  // no condition => vacuously true OR
-                              // this thread's condition is true
-              EvalConditionIgnored(this, w_walk->waitp->cond)) {
-            if (w == nullptr) {
-              w_walk->wake = true;  // can wake this waiter
-              w = w_walk;
-              pw = pw_walk;
-              if (w_walk->waitp->how == kExclusive) {
-                wr_wait = kMuWrWait;
-                break;  // bail if waking this writer
-              }
-            } else if (w_walk->waitp->how == kShared) {  // wake if a reader
-              w_walk->wake = true;
-            } else {  // writer with true condition
-              wr_wait = kMuWrWait;
-            }
-          }
-          if (w_walk->wake) {  // we're waking reader w_walk
-            pw_walk = w_walk;  // don't skip similar waiters
-          } else {             // not waking; skip as much as possible
-            pw_walk = Skip(w_walk);
-          }
-          // If pw_walk == h, then load of pw_walk->next can race with
-          // concurrent write in Enqueue(). However, at the same time
-          // we do not need to do the load, because we will bail out
-          // from the loop anyway.
-          if (pw_walk != h) {
-            w_walk = pw_walk->next;
-          }
-        }
-
-        continue;  // restart for(;;)-loop to wakeup w or to find more waiters
-      }
-      ABSL_RAW_CHECK(pw->next == w, "pw not w's predecessor");
-      // The first (and perhaps only) waiter we've chosen to wake is w, whose
-      // predecessor is pw.  If w is a reader, we must wake all the other
-      // waiters with wake==true as well.  We may also need to queue
-      // ourselves if waitp != null.  The spinlock and the lock are still
-      // held.
-
-      // This traverses the list in [ pw->next, h ], where h is the head,
-      // removing all elements with wake==true and placing them in the
-      // singly-linked list wake_list.  Returns the new head.
-      h = DequeueAllWakeable(h, pw, &wake_list);
-
-      intptr_t nv = (v & kMuEvent) | kMuDesig;
-      // assume no waiters left,
-      // set kMuDesig for INV1a
-
-      if (waitp != nullptr) {  // we must queue ourselves and sleep
-        h = Enqueue(h, waitp, v, kMuIsCond);
-        // h is new last waiter; could be null if we queued ourselves on a
-        // CondVar
-      }
-
-      ABSL_RAW_CHECK(wake_list != kPerThreadSynchNull,
-                     "unexpected empty wake list");
-
-      if (h != nullptr) {  // there are waiters left
-        h->readers = 0;
-        h->maybe_unlocking = false;  // finished unlocking
-        nv |= wr_wait | kMuWait | reinterpret_cast<intptr_t>(h);
-      }
-
-      // release both spinlock & lock
-      // can release with a store because there were waiters
-      mu_.store(nv, std::memory_order_release);
-      break;  // out of for(;;)-loop
-    }
-    // aggressive here; no one can proceed till we do
-    c = synchronization_internal::MutexDelay(c, AGGRESSIVE);
-  }  // end of for(;;)-loop
-
-  if (wake_list != kPerThreadSynchNull) {
-    int64_t total_wait_cycles = 0;
-    int64_t max_wait_cycles = 0;
-    int64_t now = CycleClock::Now();
-    do {
-      // Profile lock contention events only if the waiter was trying to acquire
-      // the lock, not waiting on a condition variable or Condition.
-      if (!wake_list->cond_waiter) {
-        int64_t cycles_waited =
-            (now - wake_list->waitp->contention_start_cycles);
-        total_wait_cycles += cycles_waited;
-        if (max_wait_cycles == 0) max_wait_cycles = cycles_waited;
-        wake_list->waitp->contention_start_cycles = now;
-        wake_list->waitp->should_submit_contention_data = true;
-      }
-      wake_list = Wakeup(wake_list);  // wake waiters
-    } while (wake_list != kPerThreadSynchNull);
-    if (total_wait_cycles > 0) {
-      mutex_tracer("slow release", this, total_wait_cycles);
-      ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0);
-      submit_profile_data(total_wait_cycles);
-      ABSL_TSAN_MUTEX_POST_DIVERT(this, 0);
-    }
-  }
-}
-
-// Used by CondVar implementation to reacquire mutex after waking from
-// condition variable.  This routine is used instead of Lock() because the
-// waiting thread may have been moved from the condition variable queue to the
-// mutex queue without a wakeup, by Trans().  In that case, when the thread is
-// finally woken, the woken thread will believe it has been woken from the
-// condition variable (i.e. its PC will be in when in the CondVar code), when
-// in fact it has just been woken from the mutex.  Thus, it must enter the slow
-// path of the mutex in the same state as if it had just woken from the mutex.
-// That is, it must ensure to clear kMuDesig (INV1b).
-void Mutex::Trans(MuHow how) {
-  this->LockSlow(how, nullptr, kMuHasBlocked | kMuIsCond);
-}
-
-// Used by CondVar implementation to effectively wake thread w from the
-// condition variable.  If this mutex is free, we simply wake the thread.
-// It will later acquire the mutex with high probability.  Otherwise, we
-// enqueue thread w on this mutex.
-void Mutex::Fer(PerThreadSynch* w) {
-  SchedulingGuard::ScopedDisable disable_rescheduling;
-  int c = 0;
-  ABSL_RAW_CHECK(w->waitp->cond == nullptr,
-                 "Mutex::Fer while waiting on Condition");
-  ABSL_RAW_CHECK(w->waitp->cv_word == nullptr,
-                 "Mutex::Fer with pending CondVar queueing");
-  // The CondVar timeout is not relevant for the Mutex wait.
-  w->waitp->timeout = {};
-  for (;;) {
-    intptr_t v = mu_.load(std::memory_order_relaxed);
-    // Note: must not queue if the mutex is unlocked (nobody will wake it).
-    // For example, we can have only kMuWait (conditional) or maybe
-    // kMuWait|kMuWrWait.
-    // conflicting != 0 implies that the waking thread cannot currently take
-    // the mutex, which in turn implies that someone else has it and can wake
-    // us if we queue.
-    const intptr_t conflicting =
-        kMuWriter | (w->waitp->how == kShared ? 0 : kMuReader);
-    if ((v & conflicting) == 0) {
-      w->next = nullptr;
-      w->state.store(PerThreadSynch::kAvailable, std::memory_order_release);
-      IncrementSynchSem(this, w);
-      return;
-    } else {
-      if ((v & (kMuSpin | kMuWait)) == 0) {  // no waiters
-        // This thread tries to become the one and only waiter.
-        PerThreadSynch* new_h =
-            Enqueue(nullptr, w->waitp, v, kMuIsCond | kMuIsFer);
-        ABSL_RAW_CHECK(new_h != nullptr,
-                       "Enqueue failed");  // we must queue ourselves
-        if (mu_.compare_exchange_strong(
-                v, reinterpret_cast<intptr_t>(new_h) | (v & kMuLow) | kMuWait,
-                std::memory_order_release, std::memory_order_relaxed)) {
-          return;
-        }
-      } else if ((v & kMuSpin) == 0 &&
-                 mu_.compare_exchange_strong(v, v | kMuSpin | kMuWait)) {
-        PerThreadSynch* h = GetPerThreadSynch(v);
-        PerThreadSynch* new_h = Enqueue(h, w->waitp, v, kMuIsCond | kMuIsFer);
-        ABSL_RAW_CHECK(new_h != nullptr,
-                       "Enqueue failed");  // we must queue ourselves
-        do {
-          v = mu_.load(std::memory_order_relaxed);
-        } while (!mu_.compare_exchange_weak(
-            v,
-            (v & kMuLow & ~kMuSpin) | kMuWait |
-                reinterpret_cast<intptr_t>(new_h),
-            std::memory_order_release, std::memory_order_relaxed));
-        return;
-      }
-    }
-    c = synchronization_internal::MutexDelay(c, GENTLE);
-  }
-}
-
-void Mutex::AssertHeld() const {
-  if ((mu_.load(std::memory_order_relaxed) & kMuWriter) == 0) {
-    SynchEvent* e = GetSynchEvent(this);
-    ABSL_RAW_LOG(FATAL, "thread should hold write lock on Mutex %p %s",
-                 static_cast<const void*>(this), (e == nullptr ? "" : e->name));
-  }
-}
-
-void Mutex::AssertReaderHeld() const {
-  if ((mu_.load(std::memory_order_relaxed) & (kMuReader | kMuWriter)) == 0) {
-    SynchEvent* e = GetSynchEvent(this);
-    ABSL_RAW_LOG(FATAL,
-                 "thread should hold at least a read lock on Mutex %p %s",
-                 static_cast<const void*>(this), (e == nullptr ? "" : e->name));
-  }
-}
-
-// -------------------------------- condition variables
-static const intptr_t kCvSpin = 0x0001L;   // spinlock protects waiter list
-static const intptr_t kCvEvent = 0x0002L;  // record events
-
-static const intptr_t kCvLow = 0x0003L;  // low order bits of CV
-
-// Hack to make constant values available to gdb pretty printer
-enum {
-  kGdbCvSpin = kCvSpin,
-  kGdbCvEvent = kCvEvent,
-  kGdbCvLow = kCvLow,
-};
-
-static_assert(PerThreadSynch::kAlignment > kCvLow,
-              "PerThreadSynch::kAlignment must be greater than kCvLow");
-
-void CondVar::EnableDebugLog(const char* name) {
-  SynchEvent* e = EnsureSynchEvent(&this->cv_, name, kCvEvent, kCvSpin);
-  e->log = true;
-  UnrefSynchEvent(e);
-}
-
-// Remove thread s from the list of waiters on this condition variable.
-void CondVar::Remove(PerThreadSynch* s) {
-  SchedulingGuard::ScopedDisable disable_rescheduling;
-  intptr_t v;
-  int c = 0;
-  for (v = cv_.load(std::memory_order_relaxed);;
-       v = cv_.load(std::memory_order_relaxed)) {
-    if ((v & kCvSpin) == 0 &&  // attempt to acquire spinlock
-        cv_.compare_exchange_strong(v, v | kCvSpin, std::memory_order_acquire,
-                                    std::memory_order_relaxed)) {
-      PerThreadSynch* h = reinterpret_cast<PerThreadSynch*>(v & ~kCvLow);
-      if (h != nullptr) {
-        PerThreadSynch* w = h;
-        while (w->next != s && w->next != h) {  // search for thread
-          w = w->next;
-        }
-        if (w->next == s) {  // found thread; remove it
-          w->next = s->next;
-          if (h == s) {
-            h = (w == s) ? nullptr : w;
-          }
-          s->next = nullptr;
-          s->state.store(PerThreadSynch::kAvailable, std::memory_order_release);
-        }
-      }
-      // release spinlock
-      cv_.store((v & kCvEvent) | reinterpret_cast<intptr_t>(h),
-                std::memory_order_release);
-      return;
-    } else {
-      // try again after a delay
-      c = synchronization_internal::MutexDelay(c, GENTLE);
-    }
-  }
-}
-
-// Queue thread waitp->thread on condition variable word cv_word using
-// wait parameters waitp.
-// We split this into a separate routine, rather than simply doing it as part
-// of WaitCommon().  If we were to queue ourselves on the condition variable
-// before calling Mutex::UnlockSlow(), the Mutex code might be re-entered (via
-// the logging code, or via a Condition function) and might potentially attempt
-// to block this thread.  That would be a problem if the thread were already on
-// a condition variable waiter queue.  Thus, we use the waitp->cv_word to tell
-// the unlock code to call CondVarEnqueue() to queue the thread on the condition
-// variable queue just before the mutex is to be unlocked, and (most
-// importantly) after any call to an external routine that might re-enter the
-// mutex code.
-static void CondVarEnqueue(SynchWaitParams* waitp) {
-  // This thread might be transferred to the Mutex queue by Fer() when
-  // we are woken.  To make sure that is what happens, Enqueue() doesn't
-  // call CondVarEnqueue() again but instead uses its normal code.  We
-  // must do this before we queue ourselves so that cv_word will be null
-  // when seen by the dequeuer, who may wish immediately to requeue
-  // this thread on another queue.
-  std::atomic<intptr_t>* cv_word = waitp->cv_word;
-  waitp->cv_word = nullptr;
-
-  intptr_t v = cv_word->load(std::memory_order_relaxed);
-  int c = 0;
-  while ((v & kCvSpin) != 0 ||  // acquire spinlock
-         !cv_word->compare_exchange_weak(v, v | kCvSpin,
-                                         std::memory_order_acquire,
-                                         std::memory_order_relaxed)) {
-    c = synchronization_internal::MutexDelay(c, GENTLE);
-    v = cv_word->load(std::memory_order_relaxed);
-  }
-  ABSL_RAW_CHECK(waitp->thread->waitp == nullptr, "waiting when shouldn't be");
-  waitp->thread->waitp = waitp;  // prepare ourselves for waiting
-  PerThreadSynch* h = reinterpret_cast<PerThreadSynch*>(v & ~kCvLow);
-  if (h == nullptr) {  // add this thread to waiter list
-    waitp->thread->next = waitp->thread;
-  } else {
-    waitp->thread->next = h->next;
-    h->next = waitp->thread;
-  }
-  waitp->thread->state.store(PerThreadSynch::kQueued,
-                             std::memory_order_relaxed);
-  cv_word->store((v & kCvEvent) | reinterpret_cast<intptr_t>(waitp->thread),
-                 std::memory_order_release);
-}
-
-bool CondVar::WaitCommon(Mutex* mutex, KernelTimeout t) {
-  bool rc = false;  // return value; true iff we timed-out
-
-  intptr_t mutex_v = mutex->mu_.load(std::memory_order_relaxed);
-  Mutex::MuHow mutex_how = ((mutex_v & kMuWriter) != 0) ? kExclusive : kShared;
-  ABSL_TSAN_MUTEX_PRE_UNLOCK(mutex, TsanFlags(mutex_how));
-
-  // maybe trace this call
-  intptr_t v = cv_.load(std::memory_order_relaxed);
-  cond_var_tracer("Wait", this);
-  if ((v & kCvEvent) != 0) {
-    PostSynchEvent(this, SYNCH_EV_WAIT);
-  }
-
-  // Release mu and wait on condition variable.
-  SynchWaitParams waitp(mutex_how, nullptr, t, mutex,
-                        Synch_GetPerThreadAnnotated(mutex), &cv_);
-  // UnlockSlow() will call CondVarEnqueue() just before releasing the
-  // Mutex, thus queuing this thread on the condition variable.  See
-  // CondVarEnqueue() for the reasons.
-  mutex->UnlockSlow(&waitp);
-
-  // wait for signal
-  while (waitp.thread->state.load(std::memory_order_acquire) ==
-         PerThreadSynch::kQueued) {
-    if (!Mutex::DecrementSynchSem(mutex, waitp.thread, t)) {
-      // DecrementSynchSem returned due to timeout.
-      // Now we will either (1) remove ourselves from the wait list in Remove
-      // below, in which case Remove will set thread.state = kAvailable and
-      // we will not call DecrementSynchSem again; or (2) Signal/SignalAll
-      // has removed us concurrently and is calling Wakeup, which will set
-      // thread.state = kAvailable and post to the semaphore.
-      // It's important to reset the timeout for the case (2) because otherwise
-      // we can live-lock in this loop since DecrementSynchSem will always
-      // return immediately due to timeout, but Signal/SignalAll is not
-      // necessary set thread.state = kAvailable yet (and is not scheduled
-      // due to thread priorities or other scheduler artifacts).
-      // Note this could also be resolved if Signal/SignalAll would set
-      // thread.state = kAvailable while holding the wait list spin lock.
-      // But this can't be easily done for SignalAll since it grabs the whole
-      // wait list with a single compare-exchange and does not really grab
-      // the spin lock.
-      t = KernelTimeout::Never();
-      this->Remove(waitp.thread);
-      rc = true;
-    }
-  }
-
-  ABSL_RAW_CHECK(waitp.thread->waitp != nullptr, "not waiting when should be");
-  waitp.thread->waitp = nullptr;  // cleanup
-
-  // maybe trace this call
-  cond_var_tracer("Unwait", this);
-  if ((v & kCvEvent) != 0) {
-    PostSynchEvent(this, SYNCH_EV_WAIT_RETURNING);
-  }
-
-  // From synchronization point of view Wait is unlock of the mutex followed
-  // by lock of the mutex. We've annotated start of unlock in the beginning
-  // of the function. Now, finish unlock and annotate lock of the mutex.
-  // (Trans is effectively lock).
-  ABSL_TSAN_MUTEX_POST_UNLOCK(mutex, TsanFlags(mutex_how));
-  ABSL_TSAN_MUTEX_PRE_LOCK(mutex, TsanFlags(mutex_how));
-  mutex->Trans(mutex_how);  // Reacquire mutex
-  ABSL_TSAN_MUTEX_POST_LOCK(mutex, TsanFlags(mutex_how), 0);
-  return rc;
-}
-
-void CondVar::Signal() {
-  SchedulingGuard::ScopedDisable disable_rescheduling;
-  ABSL_TSAN_MUTEX_PRE_SIGNAL(nullptr, 0);
-  intptr_t v;
-  int c = 0;
-  for (v = cv_.load(std::memory_order_relaxed); v != 0;
-       v = cv_.load(std::memory_order_relaxed)) {
-    if ((v & kCvSpin) == 0 &&  // attempt to acquire spinlock
-        cv_.compare_exchange_strong(v, v | kCvSpin, std::memory_order_acquire,
-                                    std::memory_order_relaxed)) {
-      PerThreadSynch* h = reinterpret_cast<PerThreadSynch*>(v & ~kCvLow);
-      PerThreadSynch* w = nullptr;
-      if (h != nullptr) {  // remove first waiter
-        w = h->next;
-        if (w == h) {
-          h = nullptr;
-        } else {
-          h->next = w->next;
-        }
-      }
-      // release spinlock
-      cv_.store((v & kCvEvent) | reinterpret_cast<intptr_t>(h),
-                std::memory_order_release);
-      if (w != nullptr) {
-        w->waitp->cvmu->Fer(w);  // wake waiter, if there was one
-        cond_var_tracer("Signal wakeup", this);
-      }
-      if ((v & kCvEvent) != 0) {
-        PostSynchEvent(this, SYNCH_EV_SIGNAL);
-      }
-      ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0);
-      return;
-    } else {
-      c = synchronization_internal::MutexDelay(c, GENTLE);
-    }
-  }
-  ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0);
-}
-
-void CondVar::SignalAll() {
-  ABSL_TSAN_MUTEX_PRE_SIGNAL(nullptr, 0);
-  intptr_t v;
-  int c = 0;
-  for (v = cv_.load(std::memory_order_relaxed); v != 0;
-       v = cv_.load(std::memory_order_relaxed)) {
-    // empty the list if spinlock free
-    // We do this by simply setting the list to empty using
-    // compare and swap.   We then have the entire list in our hands,
-    // which cannot be changing since we grabbed it while no one
-    // held the lock.
-    if ((v & kCvSpin) == 0 &&
-        cv_.compare_exchange_strong(v, v & kCvEvent, std::memory_order_acquire,
-                                    std::memory_order_relaxed)) {
-      PerThreadSynch* h = reinterpret_cast<PerThreadSynch*>(v & ~kCvLow);
-      if (h != nullptr) {
-        PerThreadSynch* w;
-        PerThreadSynch* n = h->next;
-        do {  // for every thread, wake it up
-          w = n;
-          n = n->next;
-          w->waitp->cvmu->Fer(w);
-        } while (w != h);
-        cond_var_tracer("SignalAll wakeup", this);
-      }
-      if ((v & kCvEvent) != 0) {
-        PostSynchEvent(this, SYNCH_EV_SIGNALALL);
-      }
-      ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0);
-      return;
-    } else {
-      // try again after a delay
-      c = synchronization_internal::MutexDelay(c, GENTLE);
-    }
-  }
-  ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0);
-}
-
-void ReleasableMutexLock::Release() {
-  ABSL_RAW_CHECK(this->mu_ != nullptr,
-                 "ReleasableMutexLock::Release may only be called once");
-  this->mu_->unlock();
-  this->mu_ = nullptr;
-}
-
-#ifdef ABSL_HAVE_THREAD_SANITIZER
-extern "C" void __tsan_read1(void* addr);
-#else
-#define __tsan_read1(addr)  // do nothing if TSan not enabled
-#endif
-
-// A function that just returns its argument, dereferenced
-static bool Dereference(void* arg) {
-  // ThreadSanitizer does not instrument this file for memory accesses.
-  // This function dereferences a user variable that can participate
-  // in a data race, so we need to manually tell TSan about this memory access.
-  __tsan_read1(arg);
-  return *(static_cast<bool*>(arg));
-}
-
-ABSL_CONST_INIT const Condition Condition::kTrue;
-
-Condition::Condition(bool (*func)(void*), void* arg)
-    : eval_(&CallVoidPtrFunction), arg_(arg) {
-  static_assert(sizeof(&func) <= sizeof(callback_),
-                "An overlarge function pointer passed to Condition.");
-  StoreCallback(func);
-}
-
-bool Condition::CallVoidPtrFunction(const Condition* c) {
-  using FunctionPointer = bool (*)(void*);
-  FunctionPointer function_pointer;
-  std::memcpy(&function_pointer, c->callback_, sizeof(function_pointer));
-  return (*function_pointer)(c->arg_);
-}
-
-Condition::Condition(const bool* cond)
-    : eval_(CallVoidPtrFunction),
-      // const_cast is safe since Dereference does not modify arg
-      arg_(const_cast<bool*>(cond)) {
-  using FunctionPointer = bool (*)(void*);
-  const FunctionPointer dereference = Dereference;
-  StoreCallback(dereference);
-}
-
-bool Condition::Eval() const { return (*this->eval_)(this); }
-
-bool Condition::GuaranteedEqual(const Condition* a, const Condition* b) {
-  if (a == nullptr || b == nullptr) {
-    return a == b;
-  }
-  // Check equality of the representative fields.
-  return a->eval_ == b->eval_ && a->arg_ == b->arg_ &&
-         !memcmp(a->callback_, b->callback_, sizeof(a->callback_));
-}
-
-ABSL_NAMESPACE_END
-}  // namespace absl