grpc 1.31.1 → 1.32.0.pre1

data/third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.cc

@@ -0,0 +1,106 @@
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file is a no-op if the required LowLevelAlloc support is missing.
+#include "absl/base/internal/low_level_alloc.h"
+#ifndef ABSL_LOW_LEVEL_ALLOC_MISSING
+
+#include "absl/synchronization/internal/per_thread_sem.h"
+
+#include <atomic>
+
+#include "absl/base/attributes.h"
+#include "absl/base/internal/thread_identity.h"
+#include "absl/synchronization/internal/waiter.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+void PerThreadSem::SetThreadBlockedCounter(std::atomic<int> *counter) {
+  base_internal::ThreadIdentity *identity;
+  identity = GetOrCreateCurrentThreadIdentity();
+  identity->blocked_count_ptr = counter;
+}
+
+std::atomic<int> *PerThreadSem::GetThreadBlockedCounter() {
+  base_internal::ThreadIdentity *identity;
+  identity = GetOrCreateCurrentThreadIdentity();
+  return identity->blocked_count_ptr;
+}
+
+void PerThreadSem::Init(base_internal::ThreadIdentity *identity) {
+  new (Waiter::GetWaiter(identity)) Waiter();
+  identity->ticker.store(0, std::memory_order_relaxed);
+  identity->wait_start.store(0, std::memory_order_relaxed);
+  identity->is_idle.store(false, std::memory_order_relaxed);
+}
+
+void PerThreadSem::Destroy(base_internal::ThreadIdentity *identity) {
+  Waiter::GetWaiter(identity)->~Waiter();
+}
+
+void PerThreadSem::Tick(base_internal::ThreadIdentity *identity) {
+  const int ticker =
+      identity->ticker.fetch_add(1, std::memory_order_relaxed) + 1;
+  const int wait_start = identity->wait_start.load(std::memory_order_relaxed);
+  const bool is_idle = identity->is_idle.load(std::memory_order_relaxed);
+  if (wait_start && (ticker - wait_start > Waiter::kIdlePeriods) && !is_idle) {
+    // Wakeup the waiting thread since it is time for it to become idle.
+    Waiter::GetWaiter(identity)->Poke();
+  }
+}
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+extern "C" {
+
+ABSL_ATTRIBUTE_WEAK void AbslInternalPerThreadSemPost(
+    absl::base_internal::ThreadIdentity *identity) {
+  absl::synchronization_internal::Waiter::GetWaiter(identity)->Post();
+}
+
+ABSL_ATTRIBUTE_WEAK bool AbslInternalPerThreadSemWait(
+    absl::synchronization_internal::KernelTimeout t) {
+  bool timeout = false;
+  absl::base_internal::ThreadIdentity *identity;
+  identity = absl::synchronization_internal::GetOrCreateCurrentThreadIdentity();
+
+  // Ensure wait_start != 0.
+  int ticker = identity->ticker.load(std::memory_order_relaxed);
+  identity->wait_start.store(ticker ? ticker : 1, std::memory_order_relaxed);
+  identity->is_idle.store(false, std::memory_order_relaxed);
+
+  if (identity->blocked_count_ptr != nullptr) {
+    // Increment count of threads blocked in a given thread pool.
+    identity->blocked_count_ptr->fetch_add(1, std::memory_order_relaxed);
+  }
+
+  timeout =
+      !absl::synchronization_internal::Waiter::GetWaiter(identity)->Wait(t);
+
+  if (identity->blocked_count_ptr != nullptr) {
+    identity->blocked_count_ptr->fetch_sub(1, std::memory_order_relaxed);
+  }
+
+  identity->is_idle.store(false, std::memory_order_relaxed);
+  identity->wait_start.store(0, std::memory_order_relaxed);
+  return !timeout;
+}
+
+}  // extern "C"
+
+#endif  // ABSL_LOW_LEVEL_ALLOC_MISSING

data/third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.h

@@ -0,0 +1,115 @@
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// PerThreadSem is a low-level synchronization primitive controlling the
+// runnability of a single thread, used internally by Mutex and CondVar.
+//
+// This is NOT a general-purpose synchronization mechanism, and should not be
+// used directly by applications.  Applications should use Mutex and CondVar.
+//
+// The semantics of PerThreadSem are the same as that of a counting semaphore.
+// Each thread maintains an abstract "count" value associated with its identity.
+
+#ifndef ABSL_SYNCHRONIZATION_INTERNAL_PER_THREAD_SEM_H_
+#define ABSL_SYNCHRONIZATION_INTERNAL_PER_THREAD_SEM_H_
+
+#include <atomic>
+
+#include "absl/base/internal/thread_identity.h"
+#include "absl/synchronization/internal/create_thread_identity.h"
+#include "absl/synchronization/internal/kernel_timeout.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+
+class Mutex;
+
+namespace synchronization_internal {
+
+class PerThreadSem {
+ public:
+  PerThreadSem() = delete;
+  PerThreadSem(const PerThreadSem&) = delete;
+  PerThreadSem& operator=(const PerThreadSem&) = delete;
+
+  // Routine invoked periodically (once a second) by a background thread.
+  // Has no effect on user-visible state.
+  static void Tick(base_internal::ThreadIdentity* identity);
+
+  // ---------------------------------------------------------------------------
+  // Routines used by autosizing threadpools to detect when threads are
+  // blocked.  Each thread has a counter pointer, initially zero.  If non-zero,
+  // the implementation atomically increments the counter when it blocks on a
+  // semaphore, a decrements it again when it wakes.  This allows a threadpool
+  // to keep track of how many of its threads are blocked.
+  // SetThreadBlockedCounter() should be used only by threadpool
+  // implementations.  GetThreadBlockedCounter() should be used by modules that
+  // block threads; if the pointer returned is non-zero, the location should be
+  // incremented before the thread blocks, and decremented after it wakes.
+  static void SetThreadBlockedCounter(std::atomic<int> *counter);
+  static std::atomic<int> *GetThreadBlockedCounter();
+
+ private:
+  // Create the PerThreadSem associated with "identity".  Initializes count=0.
+  // REQUIRES: May only be called by ThreadIdentity.
+  static void Init(base_internal::ThreadIdentity* identity);
+
+  // Destroy the PerThreadSem associated with "identity".
+  // REQUIRES: May only be called by ThreadIdentity.
+  static void Destroy(base_internal::ThreadIdentity* identity);
+
+  // Increments "identity"'s count.
+  static inline void Post(base_internal::ThreadIdentity* identity);
+
+  // Waits until either our count > 0 or t has expired.
+  // If count > 0, decrements count and returns true.  Otherwise returns false.
+  // !t.has_timeout() => Wait(t) will return true.
+  static inline bool Wait(KernelTimeout t);
+
+  // White-listed callers.
+  friend class PerThreadSemTest;
+  friend class absl::Mutex;
+  friend absl::base_internal::ThreadIdentity* CreateThreadIdentity();
+  friend void ReclaimThreadIdentity(void* v);
+};
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+// In some build configurations we pass --detect-odr-violations to the
+// gold linker.  This causes it to flag weak symbol overrides as ODR
+// violations.  Because ODR only applies to C++ and not C,
+// --detect-odr-violations ignores symbols not mangled with C++ names.
+// By changing our extension points to be extern "C", we dodge this
+// check.
+extern "C" {
+void AbslInternalPerThreadSemPost(
+    absl::base_internal::ThreadIdentity* identity);
+bool AbslInternalPerThreadSemWait(
+    absl::synchronization_internal::KernelTimeout t);
+}  // extern "C"
+
+void absl::synchronization_internal::PerThreadSem::Post(
+    absl::base_internal::ThreadIdentity* identity) {
+  AbslInternalPerThreadSemPost(identity);
+}
+
+bool absl::synchronization_internal::PerThreadSem::Wait(
+    absl::synchronization_internal::KernelTimeout t) {
+  return AbslInternalPerThreadSemWait(t);
+}
+
+#endif  // ABSL_SYNCHRONIZATION_INTERNAL_PER_THREAD_SEM_H_

data/third_party/abseil-cpp/absl/synchronization/internal/waiter.cc

@@ -0,0 +1,484 @@
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/synchronization/internal/waiter.h"
+
+#include "absl/base/config.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <pthread.h>
+#include <sys/time.h>
+#include <unistd.h>
+#endif
+
+#ifdef __linux__
+#include <linux/futex.h>
+#include <sys/syscall.h>
+#endif
+
+#ifdef ABSL_HAVE_SEMAPHORE_H
+#include <semaphore.h>
+#endif
+
+#include <errno.h>
+#include <stdio.h>
+#include <time.h>
+
+#include <atomic>
+#include <cassert>
+#include <cstdint>
+#include <new>
+#include <type_traits>
+
+#include "absl/base/internal/raw_logging.h"
+#include "absl/base/internal/thread_identity.h"
+#include "absl/base/optimization.h"
+#include "absl/synchronization/internal/kernel_timeout.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+static void MaybeBecomeIdle() {
+  base_internal::ThreadIdentity *identity =
+      base_internal::CurrentThreadIdentityIfPresent();
+  assert(identity != nullptr);
+  const bool is_idle = identity->is_idle.load(std::memory_order_relaxed);
+  const int ticker = identity->ticker.load(std::memory_order_relaxed);
+  const int wait_start = identity->wait_start.load(std::memory_order_relaxed);
+  if (!is_idle && ticker - wait_start > Waiter::kIdlePeriods) {
+    identity->is_idle.store(true, std::memory_order_relaxed);
+  }
+}
+
+#if ABSL_WAITER_MODE == ABSL_WAITER_MODE_FUTEX
+
+// Some Android headers are missing these definitions even though they
+// support these futex operations.
+#ifdef __BIONIC__
+#ifndef SYS_futex
+#define SYS_futex __NR_futex
+#endif
+#ifndef FUTEX_WAIT_BITSET
+#define FUTEX_WAIT_BITSET 9
+#endif
+#ifndef FUTEX_PRIVATE_FLAG
+#define FUTEX_PRIVATE_FLAG 128
+#endif
+#ifndef FUTEX_CLOCK_REALTIME
+#define FUTEX_CLOCK_REALTIME 256
+#endif
+#ifndef FUTEX_BITSET_MATCH_ANY
+#define FUTEX_BITSET_MATCH_ANY 0xFFFFFFFF
+#endif
+#endif
+
+class Futex {
+ public:
+  static int WaitUntil(std::atomic<int32_t> *v, int32_t val,
+                       KernelTimeout t) {
+    int err = 0;
+    if (t.has_timeout()) {
+      // https://locklessinc.com/articles/futex_cheat_sheet/
+      // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time.
+      struct timespec abs_timeout = t.MakeAbsTimespec();
+      // Atomically check that the futex value is still 0, and if it
+      // is, sleep until abs_timeout or until woken by FUTEX_WAKE.
+      err = syscall(
+          SYS_futex, reinterpret_cast<int32_t *>(v),
+          FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, val,
+          &abs_timeout, nullptr, FUTEX_BITSET_MATCH_ANY);
+    } else {
+      // Atomically check that the futex value is still 0, and if it
+      // is, sleep until woken by FUTEX_WAKE.
+      err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v),
+                    FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, nullptr);
+    }
+    if (err != 0) {
+      err = -errno;
+    }
+    return err;
+  }
+
+  static int Wake(std::atomic<int32_t> *v, int32_t count) {
+    int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v),
+                      FUTEX_WAKE | FUTEX_PRIVATE_FLAG, count);
+    if (ABSL_PREDICT_FALSE(err < 0)) {
+      err = -errno;
+    }
+    return err;
+  }
+};
+
+Waiter::Waiter() {
+  futex_.store(0, std::memory_order_relaxed);
+}
+
+Waiter::~Waiter() = default;
+
+bool Waiter::Wait(KernelTimeout t) {
+  // Loop until we can atomically decrement futex from a positive
+  // value, waiting on a futex while we believe it is zero.
+  // Note that, since the thread ticker is just reset, we don't need to check
+  // whether the thread is idle on the very first pass of the loop.
+  bool first_pass = true;
+  while (true) {
+    int32_t x = futex_.load(std::memory_order_relaxed);
+    while (x != 0) {
+      if (!futex_.compare_exchange_weak(x, x - 1,
+                                        std::memory_order_acquire,
+                                        std::memory_order_relaxed)) {
+        continue;  // Raced with someone, retry.
+      }
+      return true;  // Consumed a wakeup, we are done.
+    }
+
+
+    if (!first_pass) MaybeBecomeIdle();
+    const int err = Futex::WaitUntil(&futex_, 0, t);
+    if (err != 0) {
+      if (err == -EINTR || err == -EWOULDBLOCK) {
+        // Do nothing, the loop will retry.
+      } else if (err == -ETIMEDOUT) {
+        return false;
+      } else {
+        ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err);
+      }
+    }
+    first_pass = false;
+  }
+}
+
+void Waiter::Post() {
+  if (futex_.fetch_add(1, std::memory_order_release) == 0) {
+    // We incremented from 0, need to wake a potential waiter.
+    Poke();
+  }
+}
+
+void Waiter::Poke() {
+  // Wake one thread waiting on the futex.
+  const int err = Futex::Wake(&futex_, 1);
+  if (ABSL_PREDICT_FALSE(err < 0)) {
+    ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err);
+  }
+}
+
+#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_CONDVAR
+
+class PthreadMutexHolder {
+ public:
+  explicit PthreadMutexHolder(pthread_mutex_t *mu) : mu_(mu) {
+    const int err = pthread_mutex_lock(mu_);
+    if (err != 0) {
+      ABSL_RAW_LOG(FATAL, "pthread_mutex_lock failed: %d", err);
+    }
+  }
+
+  PthreadMutexHolder(const PthreadMutexHolder &rhs) = delete;
+  PthreadMutexHolder &operator=(const PthreadMutexHolder &rhs) = delete;
+
+  ~PthreadMutexHolder() {
+    const int err = pthread_mutex_unlock(mu_);
+    if (err != 0) {
+      ABSL_RAW_LOG(FATAL, "pthread_mutex_unlock failed: %d", err);
+    }
+  }
+
+ private:
+  pthread_mutex_t *mu_;
+};
+
+Waiter::Waiter() {
+  const int err = pthread_mutex_init(&mu_, 0);
+  if (err != 0) {
+    ABSL_RAW_LOG(FATAL, "pthread_mutex_init failed: %d", err);
+  }
+
+  const int err2 = pthread_cond_init(&cv_, 0);
+  if (err2 != 0) {
+    ABSL_RAW_LOG(FATAL, "pthread_cond_init failed: %d", err2);
+  }
+
+  waiter_count_ = 0;
+  wakeup_count_ = 0;
+}
+
+Waiter::~Waiter() {
+  const int err = pthread_mutex_destroy(&mu_);
+  if (err != 0) {
+    ABSL_RAW_LOG(FATAL, "pthread_mutex_destroy failed: %d", err);
+  }
+
+  const int err2 = pthread_cond_destroy(&cv_);
+  if (err2 != 0) {
+    ABSL_RAW_LOG(FATAL, "pthread_cond_destroy failed: %d", err2);
+  }
+}
+
+bool Waiter::Wait(KernelTimeout t) {
+  struct timespec abs_timeout;
+  if (t.has_timeout()) {
+    abs_timeout = t.MakeAbsTimespec();
+  }
+
+  PthreadMutexHolder h(&mu_);
+  ++waiter_count_;
+  // Loop until we find a wakeup to consume or timeout.
+  // Note that, since the thread ticker is just reset, we don't need to check
+  // whether the thread is idle on the very first pass of the loop.
+  bool first_pass = true;
+  while (wakeup_count_ == 0) {
+    if (!first_pass) MaybeBecomeIdle();
+    // No wakeups available, time to wait.
+    if (!t.has_timeout()) {
+      const int err = pthread_cond_wait(&cv_, &mu_);
+      if (err != 0) {
+        ABSL_RAW_LOG(FATAL, "pthread_cond_wait failed: %d", err);
+      }
+    } else {
+      const int err = pthread_cond_timedwait(&cv_, &mu_, &abs_timeout);
+      if (err == ETIMEDOUT) {
+        --waiter_count_;
+        return false;
+      }
+      if (err != 0) {
+        ABSL_RAW_LOG(FATAL, "pthread_cond_timedwait failed: %d", err);
+      }
+    }
+    first_pass = false;
+  }
+  // Consume a wakeup and we're done.
+  --wakeup_count_;
+  --waiter_count_;
+  return true;
+}
+
+void Waiter::Post() {
+  PthreadMutexHolder h(&mu_);
+  ++wakeup_count_;
+  InternalCondVarPoke();
+}
+
+void Waiter::Poke() {
+  PthreadMutexHolder h(&mu_);
+  InternalCondVarPoke();
+}
+
+void Waiter::InternalCondVarPoke() {
+  if (waiter_count_ != 0) {
+    const int err = pthread_cond_signal(&cv_);
+    if (ABSL_PREDICT_FALSE(err != 0)) {
+      ABSL_RAW_LOG(FATAL, "pthread_cond_signal failed: %d", err);
+    }
+  }
+}
+
+#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_SEM
+
+Waiter::Waiter() {
+  if (sem_init(&sem_, 0, 0) != 0) {
+    ABSL_RAW_LOG(FATAL, "sem_init failed with errno %d\n", errno);
+  }
+  wakeups_.store(0, std::memory_order_relaxed);
+}
+
+Waiter::~Waiter() {
+  if (sem_destroy(&sem_) != 0) {
+    ABSL_RAW_LOG(FATAL, "sem_destroy failed with errno %d\n", errno);
+  }
+}
+
+bool Waiter::Wait(KernelTimeout t) {
+  struct timespec abs_timeout;
+  if (t.has_timeout()) {
+    abs_timeout = t.MakeAbsTimespec();
+  }
+
+  // Loop until we timeout or consume a wakeup.
+  // Note that, since the thread ticker is just reset, we don't need to check
+  // whether the thread is idle on the very first pass of the loop.
+  bool first_pass = true;
+  while (true) {
+    int x = wakeups_.load(std::memory_order_relaxed);
+    while (x != 0) {
+      if (!wakeups_.compare_exchange_weak(x, x - 1,
+                                          std::memory_order_acquire,
+                                          std::memory_order_relaxed)) {
+        continue;  // Raced with someone, retry.
+      }
+      // Successfully consumed a wakeup, we're done.
+      return true;
+    }
+
+    if (!first_pass) MaybeBecomeIdle();
+    // Nothing to consume, wait (looping on EINTR).
+    while (true) {
+      if (!t.has_timeout()) {
+        if (sem_wait(&sem_) == 0) break;
+        if (errno == EINTR) continue;
+        ABSL_RAW_LOG(FATAL, "sem_wait failed: %d", errno);
+      } else {
+        if (sem_timedwait(&sem_, &abs_timeout) == 0) break;
+        if (errno == EINTR) continue;
+        if (errno == ETIMEDOUT) return false;
+        ABSL_RAW_LOG(FATAL, "sem_timedwait failed: %d", errno);
+      }
+    }
+    first_pass = false;
+  }
+}
+
+void Waiter::Post() {
+  // Post a wakeup.
+  if (wakeups_.fetch_add(1, std::memory_order_release) == 0) {
+    // We incremented from 0, need to wake a potential waiter.
+    Poke();
+  }
+}
+
+void Waiter::Poke() {
+  if (sem_post(&sem_) != 0) {  // Wake any semaphore waiter.
+    ABSL_RAW_LOG(FATAL, "sem_post failed with errno %d\n", errno);
+  }
+}
+
+#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_WIN32
+
+class Waiter::WinHelper {
+ public:
+  static SRWLOCK *GetLock(Waiter *w) {
+    return reinterpret_cast<SRWLOCK *>(&w->mu_storage_);
+  }
+
+  static CONDITION_VARIABLE *GetCond(Waiter *w) {
+    return reinterpret_cast<CONDITION_VARIABLE *>(&w->cv_storage_);
+  }
+
+  static_assert(sizeof(SRWLOCK) == sizeof(void *),
+                "`mu_storage_` does not have the same size as SRWLOCK");
+  static_assert(alignof(SRWLOCK) == alignof(void *),
+                "`mu_storage_` does not have the same alignment as SRWLOCK");
+
+  static_assert(sizeof(CONDITION_VARIABLE) == sizeof(void *),
+                "`ABSL_CONDITION_VARIABLE_STORAGE` does not have the same size "
+                "as `CONDITION_VARIABLE`");
+  static_assert(
+      alignof(CONDITION_VARIABLE) == alignof(void *),
+      "`cv_storage_` does not have the same alignment as `CONDITION_VARIABLE`");
+
+  // The SRWLOCK and CONDITION_VARIABLE types must be trivially constructible
+  // and destructible because we never call their constructors or destructors.
+  static_assert(std::is_trivially_constructible<SRWLOCK>::value,
+                "The `SRWLOCK` type must be trivially constructible");
+  static_assert(
+      std::is_trivially_constructible<CONDITION_VARIABLE>::value,
+      "The `CONDITION_VARIABLE` type must be trivially constructible");
+  static_assert(std::is_trivially_destructible<SRWLOCK>::value,
+                "The `SRWLOCK` type must be trivially destructible");
+  static_assert(std::is_trivially_destructible<CONDITION_VARIABLE>::value,
+                "The `CONDITION_VARIABLE` type must be trivially destructible");
+};
+
+class LockHolder {
+ public:
+  explicit LockHolder(SRWLOCK* mu) : mu_(mu) {
+    AcquireSRWLockExclusive(mu_);
+  }
+
+  LockHolder(const LockHolder&) = delete;
+  LockHolder& operator=(const LockHolder&) = delete;
+
+  ~LockHolder() {
+    ReleaseSRWLockExclusive(mu_);
+  }
+
+ private:
+  SRWLOCK* mu_;
+};
+
+Waiter::Waiter() {
+  auto *mu = ::new (static_cast<void *>(&mu_storage_)) SRWLOCK;
+  auto *cv = ::new (static_cast<void *>(&cv_storage_)) CONDITION_VARIABLE;
+  InitializeSRWLock(mu);
+  InitializeConditionVariable(cv);
+  waiter_count_ = 0;
+  wakeup_count_ = 0;
+}
+
+// SRW locks and condition variables do not need to be explicitly destroyed.
+// https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-initializesrwlock
+// https://stackoverflow.com/questions/28975958/why-does-windows-have-no-deleteconditionvariable-function-to-go-together-with
+Waiter::~Waiter() = default;
+
+bool Waiter::Wait(KernelTimeout t) {
+  SRWLOCK *mu = WinHelper::GetLock(this);
+  CONDITION_VARIABLE *cv = WinHelper::GetCond(this);
+
+  LockHolder h(mu);
+  ++waiter_count_;
+
+  // Loop until we find a wakeup to consume or timeout.
+  // Note that, since the thread ticker is just reset, we don't need to check
+  // whether the thread is idle on the very first pass of the loop.
+  bool first_pass = true;
+  while (wakeup_count_ == 0) {
+    if (!first_pass) MaybeBecomeIdle();
+    // No wakeups available, time to wait.
+    if (!SleepConditionVariableSRW(cv, mu, t.InMillisecondsFromNow(), 0)) {
+      // GetLastError() returns a Win32 DWORD, but we assign to
+      // unsigned long to simplify the ABSL_RAW_LOG case below.  The uniform
+      // initialization guarantees this is not a narrowing conversion.
+      const unsigned long err{GetLastError()};  // NOLINT(runtime/int)
+      if (err == ERROR_TIMEOUT) {
+        --waiter_count_;
+        return false;
+      } else {
+        ABSL_RAW_LOG(FATAL, "SleepConditionVariableSRW failed: %lu", err);
+      }
+    }
+    first_pass = false;
+  }
+  // Consume a wakeup and we're done.
+  --wakeup_count_;
+  --waiter_count_;
+  return true;
+}
+
+void Waiter::Post() {
+  LockHolder h(WinHelper::GetLock(this));
+  ++wakeup_count_;
+  InternalCondVarPoke();
+}
+
+void Waiter::Poke() {
+  LockHolder h(WinHelper::GetLock(this));
+  InternalCondVarPoke();
+}
+
+void Waiter::InternalCondVarPoke() {
+  if (waiter_count_ != 0) {
+    WakeConditionVariable(WinHelper::GetCond(this));
+  }
+}
+
+#else
+#error Unknown ABSL_WAITER_MODE
+#endif
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl