grpc 1.56.2 → 1.58.0

data/third_party/abseil-cpp/absl/synchronization/internal/sem_waiter.cc

@@ -0,0 +1,122 @@
+// Copyright 2023 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/synchronization/internal/sem_waiter.h"
+
+#ifdef ABSL_INTERNAL_HAVE_SEM_WAITER
+
+#include <semaphore.h>
+
+#include <atomic>
+#include <cassert>
+#include <cstdint>
+#include <cerrno>
+
+#include "absl/base/config.h"
+#include "absl/base/internal/raw_logging.h"
+#include "absl/base/internal/thread_identity.h"
+#include "absl/base/optimization.h"
+#include "absl/synchronization/internal/kernel_timeout.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL
+constexpr char SemWaiter::kName[];
+#endif
+
+SemWaiter::SemWaiter() : wakeups_(0) {
+  if (sem_init(&sem_, 0, 0) != 0) {
+    ABSL_RAW_LOG(FATAL, "sem_init failed with errno %d\n", errno);
+  }
+}
+
+#if defined(__GLIBC__) && \
+    (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 30))
+#define ABSL_INTERNAL_HAVE_SEM_CLOCKWAIT 1
+#elif defined(__ANDROID_API__) && __ANDROID_API__ >= 30
+#define ABSL_INTERNAL_HAVE_SEM_CLOCKWAIT 1
+#endif
+
+// Calls sem_timedwait() or possibly something else like
+// sem_clockwait() depending on the platform and
+// KernelTimeout requested. The return value is the same as a call to the return
+// value to a call to sem_timedwait().
+int SemWaiter::TimedWait(KernelTimeout t) {
+  if (KernelTimeout::SupportsSteadyClock() && t.is_relative_timeout()) {
+#if defined(ABSL_INTERNAL_HAVE_SEM_CLOCKWAIT) && defined(CLOCK_MONOTONIC)
+    const auto abs_clock_timeout = t.MakeClockAbsoluteTimespec(CLOCK_MONOTONIC);
+    return sem_clockwait(&sem_, CLOCK_MONOTONIC, &abs_clock_timeout);
+#endif
+  }
+
+  const auto abs_timeout = t.MakeAbsTimespec();
+  return sem_timedwait(&sem_, &abs_timeout);
+}
+
+bool SemWaiter::Wait(KernelTimeout t) {
+  // Loop until we timeout or consume a wakeup.
+  // Note that, since the thread ticker is just reset, we don't need to check
+  // whether the thread is idle on the very first pass of the loop.
+  bool first_pass = true;
+  while (true) {
+    int x = wakeups_.load(std::memory_order_relaxed);
+    while (x != 0) {
+      if (!wakeups_.compare_exchange_weak(x, x - 1,
+                                          std::memory_order_acquire,
+                                          std::memory_order_relaxed)) {
+        continue;  // Raced with someone, retry.
+      }
+      // Successfully consumed a wakeup, we're done.
+      return true;
+    }
+
+    if (!first_pass) MaybeBecomeIdle();
+    // Nothing to consume, wait (looping on EINTR).
+    while (true) {
+      if (!t.has_timeout()) {
+        if (sem_wait(&sem_) == 0) break;
+        if (errno == EINTR) continue;
+        ABSL_RAW_LOG(FATAL, "sem_wait failed: %d", errno);
+      } else {
+        if (TimedWait(t) == 0) break;
+        if (errno == EINTR) continue;
+        if (errno == ETIMEDOUT) return false;
+        ABSL_RAW_LOG(FATAL, "SemWaiter::TimedWait() failed: %d", errno);
+      }
+    }
+    first_pass = false;
+  }
+}
+
+void SemWaiter::Post() {
+  // Post a wakeup.
+  if (wakeups_.fetch_add(1, std::memory_order_release) == 0) {
+    // We incremented from 0, need to wake a potential waiter.
+    Poke();
+  }
+}
+
+void SemWaiter::Poke() {
+  if (sem_post(&sem_) != 0) {  // Wake any semaphore waiter.
+    ABSL_RAW_LOG(FATAL, "sem_post failed with errno %d\n", errno);
+  }
+}
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_INTERNAL_HAVE_SEM_WAITER

data/third_party/abseil-cpp/absl/synchronization/internal/sem_waiter.h

@@ -0,0 +1,65 @@
+// Copyright 2023 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef ABSL_SYNCHRONIZATION_INTERNAL_SEM_WAITER_H_
+#define ABSL_SYNCHRONIZATION_INTERNAL_SEM_WAITER_H_
+
+#include "absl/base/config.h"
+
+#ifdef ABSL_HAVE_SEMAPHORE_H
+#include <semaphore.h>
+
+#include <atomic>
+#include <cstdint>
+
+#include "absl/base/internal/thread_identity.h"
+#include "absl/synchronization/internal/futex.h"
+#include "absl/synchronization/internal/kernel_timeout.h"
+#include "absl/synchronization/internal/waiter_base.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+#define ABSL_INTERNAL_HAVE_SEM_WAITER 1
+
+class SemWaiter : public WaiterCrtp<SemWaiter> {
+ public:
+  SemWaiter();
+
+  bool Wait(KernelTimeout t);
+  void Post();
+  void Poke();
+
+  static constexpr char kName[] = "SemWaiter";
+
+ private:
+  int TimedWait(KernelTimeout t);
+
+  sem_t sem_;
+
+  // This seems superfluous, but for Poke() we need to cause spurious
+  // wakeups on the semaphore. Hence we can't actually use the
+  // semaphore's count.
+  std::atomic<int> wakeups_;
+};
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_HAVE_SEMAPHORE_H
+
+#endif  // ABSL_SYNCHRONIZATION_INTERNAL_SEM_WAITER_H_

data/third_party/abseil-cpp/absl/synchronization/internal/stdcpp_waiter.cc

@@ -0,0 +1,91 @@
+// Copyright 2023 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/synchronization/internal/stdcpp_waiter.h"
+
+#ifdef ABSL_INTERNAL_HAVE_STDCPP_WAITER
+
+#include <chrono>  // NOLINT(build/c++11)
+#include <condition_variable>  // NOLINT(build/c++11)
+#include <mutex>  // NOLINT(build/c++11)
+
+#include "absl/base/config.h"
+#include "absl/base/internal/raw_logging.h"
+#include "absl/base/internal/thread_identity.h"
+#include "absl/base/optimization.h"
+#include "absl/synchronization/internal/kernel_timeout.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL
+constexpr char StdcppWaiter::kName[];
+#endif
+
+StdcppWaiter::StdcppWaiter() : waiter_count_(0), wakeup_count_(0) {}
+
+bool StdcppWaiter::Wait(KernelTimeout t) {
+  std::unique_lock<std::mutex> lock(mu_);
+  ++waiter_count_;
+
+  // Loop until we find a wakeup to consume or timeout.
+  // Note that, since the thread ticker is just reset, we don't need to check
+  // whether the thread is idle on the very first pass of the loop.
+  bool first_pass = true;
+  while (wakeup_count_ == 0) {
+    if (!first_pass) MaybeBecomeIdle();
+    // No wakeups available, time to wait.
+    if (!t.has_timeout()) {
+      cv_.wait(lock);
+    } else {
+      auto wait_result = t.SupportsSteadyClock() && t.is_relative_timeout()
+                             ? cv_.wait_for(lock, t.ToChronoDuration())
+                             : cv_.wait_until(lock, t.ToChronoTimePoint());
+      if (wait_result == std::cv_status::timeout) {
+        --waiter_count_;
+        return false;
+      }
+    }
+    first_pass = false;
+  }
+
+  // Consume a wakeup and we're done.
+  --wakeup_count_;
+  --waiter_count_;
+  return true;
+}
+
+void StdcppWaiter::Post() {
+  std::lock_guard<std::mutex> lock(mu_);
+  ++wakeup_count_;
+  InternalCondVarPoke();
+}
+
+void StdcppWaiter::Poke() {
+  std::lock_guard<std::mutex> lock(mu_);
+  InternalCondVarPoke();
+}
+
+void StdcppWaiter::InternalCondVarPoke() {
+  if (waiter_count_ != 0) {
+    cv_.notify_one();
+  }
+}
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_INTERNAL_HAVE_STDCPP_WAITER

data/third_party/abseil-cpp/absl/synchronization/internal/stdcpp_waiter.h

@@ -0,0 +1,56 @@
+// Copyright 2023 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef ABSL_SYNCHRONIZATION_INTERNAL_STDCPP_WAITER_H_
+#define ABSL_SYNCHRONIZATION_INTERNAL_STDCPP_WAITER_H_
+
+#include <condition_variable>  // NOLINT(build/c++11)
+#include <mutex>  // NOLINT(build/c++11)
+
+#include "absl/base/config.h"
+#include "absl/synchronization/internal/kernel_timeout.h"
+#include "absl/synchronization/internal/waiter_base.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+#define ABSL_INTERNAL_HAVE_STDCPP_WAITER 1
+
+class StdcppWaiter : public WaiterCrtp<StdcppWaiter> {
+ public:
+  StdcppWaiter();
+
+  bool Wait(KernelTimeout t);
+  void Post();
+  void Poke();
+
+  static constexpr char kName[] = "StdcppWaiter";
+
+ private:
+  // REQUIRES: mu_ must be held.
+  void InternalCondVarPoke();
+
+  std::mutex mu_;
+  std::condition_variable cv_;
+  int waiter_count_;
+  int wakeup_count_;  // Unclaimed wakeups.
+};
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_SYNCHRONIZATION_INTERNAL_STDCPP_WAITER_H_

data/third_party/abseil-cpp/absl/synchronization/internal/waiter.h

@@ -17,142 +17,48 @@
 #define ABSL_SYNCHRONIZATION_INTERNAL_WAITER_H_
 
 #include "absl/base/config.h"
-
-#ifdef _WIN32
-#include <sdkddkver.h>
-#else
-#include <pthread.h>
-#endif
-
-#ifdef __linux__
-#include <linux/futex.h>
-#endif
-
-#ifdef ABSL_HAVE_SEMAPHORE_H
-#include <semaphore.h>
-#endif
-
-#include <atomic>
-#include <cstdint>
-
-#include "absl/base/internal/thread_identity.h"
-#include "absl/synchronization/internal/futex.h"
-#include "absl/synchronization/internal/kernel_timeout.h"
+#include "absl/synchronization/internal/futex_waiter.h"
+#include "absl/synchronization/internal/pthread_waiter.h"
+#include "absl/synchronization/internal/sem_waiter.h"
+#include "absl/synchronization/internal/stdcpp_waiter.h"
+#include "absl/synchronization/internal/win32_waiter.h"
 
 // May be chosen at compile time via -DABSL_FORCE_WAITER_MODE=<index>
 #define ABSL_WAITER_MODE_FUTEX 0
 #define ABSL_WAITER_MODE_SEM 1
 #define ABSL_WAITER_MODE_CONDVAR 2
 #define ABSL_WAITER_MODE_WIN32 3
+#define ABSL_WAITER_MODE_STDCPP 4
 
 #if defined(ABSL_FORCE_WAITER_MODE)
 #define ABSL_WAITER_MODE ABSL_FORCE_WAITER_MODE
-#elif defined(_WIN32) && _WIN32_WINNT >= _WIN32_WINNT_VISTA
+#elif defined(ABSL_INTERNAL_HAVE_WIN32_WAITER)
 #define ABSL_WAITER_MODE ABSL_WAITER_MODE_WIN32
-#elif defined(ABSL_INTERNAL_HAVE_FUTEX)
+#elif defined(ABSL_INTERNAL_HAVE_FUTEX_WAITER)
 #define ABSL_WAITER_MODE ABSL_WAITER_MODE_FUTEX
-#elif defined(ABSL_HAVE_SEMAPHORE_H)
+#elif defined(ABSL_INTERNAL_HAVE_SEM_WAITER)
 #define ABSL_WAITER_MODE ABSL_WAITER_MODE_SEM
-#else
+#elif defined(ABSL_INTERNAL_HAVE_PTHREAD_WAITER)
 #define ABSL_WAITER_MODE ABSL_WAITER_MODE_CONDVAR
+#else
+#error ABSL_WAITER_MODE is undefined
 #endif
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
 namespace synchronization_internal {
 
-// Waiter is an OS-specific semaphore.
-class Waiter {
- public:
-  // Prepare any data to track waits.
-  Waiter();
-
-  // Not copyable or movable
-  Waiter(const Waiter&) = delete;
-  Waiter& operator=(const Waiter&) = delete;
-
-  // Blocks the calling thread until a matching call to `Post()` or
-  // `t` has passed. Returns `true` if woken (`Post()` called),
-  // `false` on timeout.
-  bool Wait(KernelTimeout t);
-
-  // Restart the caller of `Wait()` as with a normal semaphore.
-  void Post();
-
-  // If anyone is waiting, wake them up temporarily and cause them to
-  // call `MaybeBecomeIdle()`. They will then return to waiting for a
-  // `Post()` or timeout.
-  void Poke();
-
-  // Returns the Waiter associated with the identity.
-  static Waiter* GetWaiter(base_internal::ThreadIdentity* identity) {
-    static_assert(
-        sizeof(Waiter) <= sizeof(base_internal::ThreadIdentity::WaiterState),
-        "Insufficient space for Waiter");
-    return reinterpret_cast<Waiter*>(identity->waiter_state.data);
-  }
-
-  // How many periods to remain idle before releasing resources
-#ifndef ABSL_HAVE_THREAD_SANITIZER
-  static constexpr int kIdlePeriods = 60;
-#else
-  // Memory consumption under ThreadSanitizer is a serious concern,
-  // so we release resources sooner. The value of 1 leads to 1 to 2 second
-  // delay before marking a thread as idle.
-  static const int kIdlePeriods = 1;
-#endif
-
- private:
-  // The destructor must not be called since Mutex/CondVar
-  // can use PerThreadSem/Waiter after the thread exits.
-  // Waiter objects are embedded in ThreadIdentity objects,
-  // which are reused via a freelist and are never destroyed.
-  ~Waiter() = delete;
-
 #if ABSL_WAITER_MODE == ABSL_WAITER_MODE_FUTEX
-  // Futexes are defined by specification to be 32-bits.
-  // Thus std::atomic<int32_t> must be just an int32_t with lockfree methods.
-  std::atomic<int32_t> futex_;
-  static_assert(sizeof(int32_t) == sizeof(futex_), "Wrong size for futex");
-
-#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_CONDVAR
-  // REQUIRES: mu_ must be held.
-  void InternalCondVarPoke();
-
-  pthread_mutex_t mu_;
-  pthread_cond_t cv_;
-  int waiter_count_;
-  int wakeup_count_;  // Unclaimed wakeups.
-
+using Waiter = FutexWaiter;
 #elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_SEM
-  sem_t sem_;
-  // This seems superfluous, but for Poke() we need to cause spurious
-  // wakeups on the semaphore. Hence we can't actually use the
-  // semaphore's count.
-  std::atomic<int> wakeups_;
-
+using Waiter = SemWaiter;
+#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_CONDVAR
+using Waiter = PthreadWaiter;
 #elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_WIN32
-  // WinHelper - Used to define utilities for accessing the lock and
-  // condition variable storage once the types are complete.
-  class WinHelper;
-
-  // REQUIRES: WinHelper::GetLock(this) must be held.
-  void InternalCondVarPoke();
-
-  // We can't include Windows.h in our headers, so we use aligned character
-  // buffers to define the storage of SRWLOCK and CONDITION_VARIABLE.
-  // SRW locks and condition variables do not need to be explicitly destroyed.
-  // https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-initializesrwlock
-  // https://stackoverflow.com/questions/28975958/why-does-windows-have-no-deleteconditionvariable-function-to-go-together-with
-  alignas(void*) unsigned char mu_storage_[sizeof(void*)];
-  alignas(void*) unsigned char cv_storage_[sizeof(void*)];
-  int waiter_count_;
-  int wakeup_count_;
-
-#else
-  #error Unknown ABSL_WAITER_MODE
+using Waiter = Win32Waiter;
+#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_STDCPP
+using Waiter = StdcppWaiter;
 #endif
-};
 
 }  // namespace synchronization_internal
 ABSL_NAMESPACE_END

data/third_party/abseil-cpp/absl/synchronization/internal/waiter_base.cc

@@ -0,0 +1,42 @@
+// Copyright 2023 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/synchronization/internal/waiter_base.h"
+
+#include "absl/base/config.h"
+#include "absl/base/internal/thread_identity.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL
+constexpr int WaiterBase::kIdlePeriods;
+#endif
+
+void WaiterBase::MaybeBecomeIdle() {
+  base_internal::ThreadIdentity *identity =
+      base_internal::CurrentThreadIdentityIfPresent();
+  assert(identity != nullptr);
+  const bool is_idle = identity->is_idle.load(std::memory_order_relaxed);
+  const int ticker = identity->ticker.load(std::memory_order_relaxed);
+  const int wait_start = identity->wait_start.load(std::memory_order_relaxed);
+  if (!is_idle && ticker - wait_start > kIdlePeriods) {
+    identity->is_idle.store(true, std::memory_order_relaxed);
+  }
+}
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl

data/third_party/abseil-cpp/absl/synchronization/internal/waiter_base.h

@@ -0,0 +1,90 @@
+// Copyright 2023 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef ABSL_SYNCHRONIZATION_INTERNAL_WAITER_BASE_H_
+#define ABSL_SYNCHRONIZATION_INTERNAL_WAITER_BASE_H_
+
+#include "absl/base/config.h"
+#include "absl/base/internal/thread_identity.h"
+#include "absl/synchronization/internal/kernel_timeout.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+// `Waiter` is a platform specific semaphore implementation that `PerThreadSem`
+// waits on to implement blocking in `absl::Mutex`.  Implementations should
+// inherit from `WaiterCrtp` and must implement `Wait()`, `Post()`, and `Poke()`
+// as described in `WaiterBase`.  `waiter.h` selects the implementation and uses
+// static-dispatch for performance.
+class WaiterBase {
+ public:
+  WaiterBase() = default;
+
+  // Not copyable or movable
+  WaiterBase(const WaiterBase&) = delete;
+  WaiterBase& operator=(const WaiterBase&) = delete;
+
+  // Blocks the calling thread until a matching call to `Post()` or
+  // `t` has passed. Returns `true` if woken (`Post()` called),
+  // `false` on timeout.
+  //
+  // bool Wait(KernelTimeout t);
+
+  // Restart the caller of `Wait()` as with a normal semaphore.
+  //
+  // void Post();
+
+  // If anyone is waiting, wake them up temporarily and cause them to
+  // call `MaybeBecomeIdle()`. They will then return to waiting for a
+  // `Post()` or timeout.
+  //
+  // void Poke();
+
+  // Returns the name of this implementation. Used only for debugging.
+  //
+  // static constexpr char kName[];
+
+  // How many periods to remain idle before releasing resources
+#ifndef ABSL_HAVE_THREAD_SANITIZER
+  static constexpr int kIdlePeriods = 60;
+#else
+  // Memory consumption under ThreadSanitizer is a serious concern,
+  // so we release resources sooner. The value of 1 leads to 1 to 2 second
+  // delay before marking a thread as idle.
+  static constexpr int kIdlePeriods = 1;
+#endif
+
+ protected:
+  static void MaybeBecomeIdle();
+};
+
+template <typename T>
+class WaiterCrtp : public WaiterBase {
+ public:
+  // Returns the Waiter associated with the identity.
+  static T* GetWaiter(base_internal::ThreadIdentity* identity) {
+    static_assert(
+        sizeof(T) <= sizeof(base_internal::ThreadIdentity::WaiterState),
+        "Insufficient space for Waiter");
+    return reinterpret_cast<T*>(identity->waiter_state.data);
+  }
+};
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_SYNCHRONIZATION_INTERNAL_WAITER_BASE_H_