grpc 1.28.0 → 1.37.0

data/third_party/abseil-cpp/absl/synchronization/barrier.h

@@ -0,0 +1,79 @@
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// -----------------------------------------------------------------------------
+// barrier.h
+// -----------------------------------------------------------------------------
+
+#ifndef ABSL_SYNCHRONIZATION_BARRIER_H_
+#define ABSL_SYNCHRONIZATION_BARRIER_H_
+
+#include "absl/base/thread_annotations.h"
+#include "absl/synchronization/mutex.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+
+// Barrier
+//
+// This class creates a barrier which blocks threads until a prespecified
+// threshold of threads (`num_threads`) utilizes the barrier. A thread utilizes
+// the `Barrier` by calling `Block()` on the barrier, which will block that
+// thread; no call to `Block()` will return until `num_threads` threads have
+// called it.
+//
+// Exactly one call to `Block()` will return `true`, which is then responsible
+// for destroying the barrier; because stack allocation will cause the barrier
+// to be deleted when it is out of scope, barriers should not be stack
+// allocated.
+//
+// Example:
+//
+//   // Main thread creates a `Barrier`:
+//   barrier = new Barrier(num_threads);
+//
+//   // Each participating thread could then call:
+//   if (barrier->Block()) delete barrier;  // Exactly one call to `Block()`
+//                                          // returns `true`; that call
+//                                          // deletes the barrier.
+class Barrier {
+ public:
+  // `num_threads` is the number of threads that will participate in the barrier
+  explicit Barrier(int num_threads)
+      : num_to_block_(num_threads), num_to_exit_(num_threads) {}
+
+  Barrier(const Barrier&) = delete;
+  Barrier& operator=(const Barrier&) = delete;
+
+  // Barrier::Block()
+  //
+  // Blocks the current thread, and returns only when the `num_threads`
+  // threshold of threads utilizing this barrier has been reached. `Block()`
+  // returns `true` for precisely one caller, which may then destroy the
+  // barrier.
+  //
+  // Memory ordering: For any threads X and Y, any action taken by X
+  // before X calls `Block()` will be visible to Y after Y returns from
+  // `Block()`.
+  bool Block();
+
+ private:
+  Mutex lock_;
+  int num_to_block_ ABSL_GUARDED_BY(lock_);
+  int num_to_exit_ ABSL_GUARDED_BY(lock_);
+};
+
+ABSL_NAMESPACE_END
+}  // namespace absl
+#endif  // ABSL_SYNCHRONIZATION_BARRIER_H_

data/third_party/abseil-cpp/absl/synchronization/blocking_counter.cc

@@ -0,0 +1,57 @@
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/synchronization/blocking_counter.h"
+
+#include "absl/base/internal/raw_logging.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+
+// Return whether int *arg is zero.
+static bool IsZero(void *arg) {
+  return 0 == *reinterpret_cast<int *>(arg);
+}
+
+bool BlockingCounter::DecrementCount() {
+  MutexLock l(&lock_);
+  count_--;
+  if (count_ < 0) {
+    ABSL_RAW_LOG(
+        FATAL,
+        "BlockingCounter::DecrementCount() called too many times.  count=%d",
+        count_);
+  }
+  return count_ == 0;
+}
+
+void BlockingCounter::Wait() {
+  MutexLock l(&this->lock_);
+  ABSL_RAW_CHECK(count_ >= 0, "BlockingCounter underflow");
+
+  // only one thread may call Wait(). To support more than one thread,
+  // implement a counter num_to_exit, like in the Barrier class.
+  ABSL_RAW_CHECK(num_waiting_ == 0, "multiple threads called Wait()");
+  num_waiting_++;
+
+  this->lock_.Await(Condition(IsZero, &this->count_));
+
+  // At this point, We know that all threads executing DecrementCount have
+  // released the lock, and so will not touch this object again.
+  // Therefore, the thread calling this method is free to delete the object
+  // after we return from this method.
+}
+
+ABSL_NAMESPACE_END
+}  // namespace absl

data/third_party/abseil-cpp/absl/synchronization/blocking_counter.h

@@ -0,0 +1,99 @@
+//
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// -----------------------------------------------------------------------------
+// blocking_counter.h
+// -----------------------------------------------------------------------------
+
+#ifndef ABSL_SYNCHRONIZATION_BLOCKING_COUNTER_H_
+#define ABSL_SYNCHRONIZATION_BLOCKING_COUNTER_H_
+
+#include "absl/base/thread_annotations.h"
+#include "absl/synchronization/mutex.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+
+// BlockingCounter
+//
+// This class allows a thread to block for a pre-specified number of actions.
+// `BlockingCounter` maintains a single non-negative abstract integer "count"
+// with an initial value `initial_count`. A thread can then call `Wait()` on
+// this blocking counter to block until the specified number of events occur;
+// worker threads then call 'DecrementCount()` on the counter upon completion of
+// their work. Once the counter's internal "count" reaches zero, the blocked
+// thread unblocks.
+//
+// A `BlockingCounter` requires the following:
+//     - its `initial_count` is non-negative.
+//     - the number of calls to `DecrementCount()` on it is at most
+//       `initial_count`.
+//     - `Wait()` is called at most once on it.
+//
+// Given the above requirements, a `BlockingCounter` provides the following
+// guarantees:
+//     - Once its internal "count" reaches zero, no legal action on the object
+//       can further change the value of "count".
+//     - When `Wait()` returns, it is legal to destroy the `BlockingCounter`.
+//     - When `Wait()` returns, the number of calls to `DecrementCount()` on
+//       this blocking counter exactly equals `initial_count`.
+//
+// Example:
+//     BlockingCounter bcount(N);         // there are N items of work
+//     ... Allow worker threads to start.
+//     ... On completing each work item, workers do:
+//     ... bcount.DecrementCount();      // an item of work has been completed
+//
+//     bcount.Wait();                    // wait for all work to be complete
+//
+class BlockingCounter {
+ public:
+  explicit BlockingCounter(int initial_count)
+      : count_(initial_count), num_waiting_(0) {}
+
+  BlockingCounter(const BlockingCounter&) = delete;
+  BlockingCounter& operator=(const BlockingCounter&) = delete;
+
+  // BlockingCounter::DecrementCount()
+  //
+  // Decrements the counter's "count" by one, and return "count == 0". This
+  // function requires that "count != 0" when it is called.
+  //
+  // Memory ordering: For any threads X and Y, any action taken by X
+  // before it calls `DecrementCount()` is visible to thread Y after
+  // Y's call to `DecrementCount()`, provided Y's call returns `true`.
+  bool DecrementCount();
+
+  // BlockingCounter::Wait()
+  //
+  // Blocks until the counter reaches zero. This function may be called at most
+  // once. On return, `DecrementCount()` will have been called "initial_count"
+  // times and the blocking counter may be destroyed.
+  //
+  // Memory ordering: For any threads X and Y, any action taken by X
+  // before X calls `DecrementCount()` is visible to Y after Y returns
+  // from `Wait()`.
+  void Wait();
+
+ private:
+  Mutex lock_;
+  int count_ ABSL_GUARDED_BY(lock_);
+  int num_waiting_ ABSL_GUARDED_BY(lock_);
+};
+
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_SYNCHRONIZATION_BLOCKING_COUNTER_H_

data/third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.cc

@@ -0,0 +1,140 @@
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <new>
+
+// This file is a no-op if the required LowLevelAlloc support is missing.
+#include "absl/base/internal/low_level_alloc.h"
+#ifndef ABSL_LOW_LEVEL_ALLOC_MISSING
+
+#include <string.h>
+
+#include "absl/base/attributes.h"
+#include "absl/base/internal/spinlock.h"
+#include "absl/base/internal/thread_identity.h"
+#include "absl/synchronization/internal/per_thread_sem.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+// ThreadIdentity storage is persistent, we maintain a free-list of previously
+// released ThreadIdentity objects.
+ABSL_CONST_INIT static base_internal::SpinLock freelist_lock(
+    absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
+ABSL_CONST_INIT static base_internal::ThreadIdentity* thread_identity_freelist;
+
+// A per-thread destructor for reclaiming associated ThreadIdentity objects.
+// Since we must preserve their storage we cache them for re-use.
+void ReclaimThreadIdentity(void* v) {
+  base_internal::ThreadIdentity* identity =
+      static_cast<base_internal::ThreadIdentity*>(v);
+
+  // all_locks might have been allocated by the Mutex implementation.
+  // We free it here when we are notified that our thread is dying.
+  if (identity->per_thread_synch.all_locks != nullptr) {
+    base_internal::LowLevelAlloc::Free(identity->per_thread_synch.all_locks);
+  }
+
+  PerThreadSem::Destroy(identity);
+
+  // We must explicitly clear the current thread's identity:
+  // (a) Subsequent (unrelated) per-thread destructors may require an identity.
+  //     We must guarantee a new identity is used in this case (this instructor
+  //     will be reinvoked up to PTHREAD_DESTRUCTOR_ITERATIONS in this case).
+  // (b) ThreadIdentity implementations may depend on memory that is not
+  //     reinitialized before reuse.  We must allow explicit clearing of the
+  //     association state in this case.
+  base_internal::ClearCurrentThreadIdentity();
+  {
+    base_internal::SpinLockHolder l(&freelist_lock);
+    identity->next = thread_identity_freelist;
+    thread_identity_freelist = identity;
+  }
+}
+
+// Return value rounded up to next multiple of align.
+// Align must be a power of two.
+static intptr_t RoundUp(intptr_t addr, intptr_t align) {
+  return (addr + align - 1) & ~(align - 1);
+}
+
+static void ResetThreadIdentity(base_internal::ThreadIdentity* identity) {
+  base_internal::PerThreadSynch* pts = &identity->per_thread_synch;
+  pts->next = nullptr;
+  pts->skip = nullptr;
+  pts->may_skip = false;
+  pts->waitp = nullptr;
+  pts->suppress_fatal_errors = false;
+  pts->readers = 0;
+  pts->priority = 0;
+  pts->next_priority_read_cycles = 0;
+  pts->state.store(base_internal::PerThreadSynch::State::kAvailable,
+                   std::memory_order_relaxed);
+  pts->maybe_unlocking = false;
+  pts->wake = false;
+  pts->cond_waiter = false;
+  pts->all_locks = nullptr;
+  identity->blocked_count_ptr = nullptr;
+  identity->ticker.store(0, std::memory_order_relaxed);
+  identity->wait_start.store(0, std::memory_order_relaxed);
+  identity->is_idle.store(false, std::memory_order_relaxed);
+  identity->next = nullptr;
+}
+
+static base_internal::ThreadIdentity* NewThreadIdentity() {
+  base_internal::ThreadIdentity* identity = nullptr;
+
+  {
+    // Re-use a previously released object if possible.
+    base_internal::SpinLockHolder l(&freelist_lock);
+    if (thread_identity_freelist) {
+      identity = thread_identity_freelist;  // Take list-head.
+      thread_identity_freelist = thread_identity_freelist->next;
+    }
+  }
+
+  if (identity == nullptr) {
+    // Allocate enough space to align ThreadIdentity to a multiple of
+    // PerThreadSynch::kAlignment. This space is never released (it is
+    // added to a freelist by ReclaimThreadIdentity instead).
+    void* allocation = base_internal::LowLevelAlloc::Alloc(
+        sizeof(*identity) + base_internal::PerThreadSynch::kAlignment - 1);
+    // Round up the address to the required alignment.
+    identity = reinterpret_cast<base_internal::ThreadIdentity*>(
+        RoundUp(reinterpret_cast<intptr_t>(allocation),
+                base_internal::PerThreadSynch::kAlignment));
+  }
+  ResetThreadIdentity(identity);
+
+  return identity;
+}
+
+// Allocates and attaches ThreadIdentity object for the calling thread.  Returns
+// the new identity.
+// REQUIRES: CurrentThreadIdentity(false) == nullptr
+base_internal::ThreadIdentity* CreateThreadIdentity() {
+  base_internal::ThreadIdentity* identity = NewThreadIdentity();
+  PerThreadSem::Init(identity);
+  // Associate the value with the current thread, and attach our destructor.
+  base_internal::SetCurrentThreadIdentity(identity, ReclaimThreadIdentity);
+  return identity;
+}
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_LOW_LEVEL_ALLOC_MISSING

data/third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2017 The Abseil Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Interface for getting the current ThreadIdentity, creating one if necessary.
+// See thread_identity.h.
+//
+// This file is separate from thread_identity.h because creating a new
+// ThreadIdentity requires slightly higher level libraries (per_thread_sem
+// and low_level_alloc) than accessing an existing one.  This separation allows
+// us to have a smaller //absl/base:base.
+
+#ifndef ABSL_SYNCHRONIZATION_INTERNAL_CREATE_THREAD_IDENTITY_H_
+#define ABSL_SYNCHRONIZATION_INTERNAL_CREATE_THREAD_IDENTITY_H_
+
+#include "absl/base/internal/thread_identity.h"
+#include "absl/base/port.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+// Allocates and attaches a ThreadIdentity object for the calling thread.
+// For private use only.
+base_internal::ThreadIdentity* CreateThreadIdentity();
+
+// A per-thread destructor for reclaiming associated ThreadIdentity objects.
+// For private use only.
+void ReclaimThreadIdentity(void* v);
+
+// Returns the ThreadIdentity object representing the calling thread; guaranteed
+// to be unique for its lifetime.  The returned object will remain valid for the
+// program's lifetime; although it may be re-assigned to a subsequent thread.
+// If one does not exist for the calling thread, allocate it now.
+inline base_internal::ThreadIdentity* GetOrCreateCurrentThreadIdentity() {
+  base_internal::ThreadIdentity* identity =
+      base_internal::CurrentThreadIdentityIfPresent();
+  if (ABSL_PREDICT_FALSE(identity == nullptr)) {
+    return CreateThreadIdentity();
+  }
+  return identity;
+}
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_SYNCHRONIZATION_INTERNAL_CREATE_THREAD_IDENTITY_H_

data/third_party/abseil-cpp/absl/synchronization/internal/graphcycles.cc

@@ -0,0 +1,698 @@
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// GraphCycles provides incremental cycle detection on a dynamic
+// graph using the following algorithm:
+//
+// A dynamic topological sort algorithm for directed acyclic graphs
+// David J. Pearce, Paul H. J. Kelly
+// Journal of Experimental Algorithmics (JEA) JEA Homepage archive
+// Volume 11, 2006, Article No. 1.7
+//
+// Brief summary of the algorithm:
+//
+// (1) Maintain a rank for each node that is consistent
+//     with the topological sort of the graph. I.e., path from x to y
+//     implies rank[x] < rank[y].
+// (2) When a new edge (x->y) is inserted, do nothing if rank[x] < rank[y].
+// (3) Otherwise: adjust ranks in the neighborhood of x and y.
+
+#include "absl/base/attributes.h"
+// This file is a no-op if the required LowLevelAlloc support is missing.
+#include "absl/base/internal/low_level_alloc.h"
+#ifndef ABSL_LOW_LEVEL_ALLOC_MISSING
+
+#include "absl/synchronization/internal/graphcycles.h"
+
+#include <algorithm>
+#include <array>
+#include <limits>
+#include "absl/base/internal/hide_ptr.h"
+#include "absl/base/internal/raw_logging.h"
+#include "absl/base/internal/spinlock.h"
+
+// Do not use STL.   This module does not use standard memory allocation.
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+namespace {
+
+// Avoid LowLevelAlloc's default arena since it calls malloc hooks in
+// which people are doing things like acquiring Mutexes.
+ABSL_CONST_INIT static absl::base_internal::SpinLock arena_mu(
+    absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
+ABSL_CONST_INIT static base_internal::LowLevelAlloc::Arena* arena;
+
+static void InitArenaIfNecessary() {
+  arena_mu.Lock();
+  if (arena == nullptr) {
+    arena = base_internal::LowLevelAlloc::NewArena(0);
+  }
+  arena_mu.Unlock();
+}
+
+// Number of inlined elements in Vec.  Hash table implementation
+// relies on this being a power of two.
+static const uint32_t kInline = 8;
+
+// A simple LowLevelAlloc based resizable vector with inlined storage
+// for a few elements.  T must be a plain type since constructor
+// and destructor are not run on elements of type T managed by Vec.
+template <typename T>
+class Vec {
+ public:
+  Vec() { Init(); }
+  ~Vec() { Discard(); }
+
+  void clear() {
+    Discard();
+    Init();
+  }
+
+  bool empty() const { return size_ == 0; }
+  uint32_t size() const { return size_; }
+  T* begin() { return ptr_; }
+  T* end() { return ptr_ + size_; }
+  const T& operator[](uint32_t i) const { return ptr_[i]; }
+  T& operator[](uint32_t i) { return ptr_[i]; }
+  const T& back() const { return ptr_[size_-1]; }
+  void pop_back() { size_--; }
+
+  void push_back(const T& v) {
+    if (size_ == capacity_) Grow(size_ + 1);
+    ptr_[size_] = v;
+    size_++;
+  }
+
+  void resize(uint32_t n) {
+    if (n > capacity_) Grow(n);
+    size_ = n;
+  }
+
+  void fill(const T& val) {
+    for (uint32_t i = 0; i < size(); i++) {
+      ptr_[i] = val;
+    }
+  }
+
+  // Guarantees src is empty at end.
+  // Provided for the hash table resizing code below.
+  void MoveFrom(Vec<T>* src) {
+    if (src->ptr_ == src->space_) {
+      // Need to actually copy
+      resize(src->size_);
+      std::copy(src->ptr_, src->ptr_ + src->size_, ptr_);
+      src->size_ = 0;
+    } else {
+      Discard();
+      ptr_ = src->ptr_;
+      size_ = src->size_;
+      capacity_ = src->capacity_;
+      src->Init();
+    }
+  }
+
+ private:
+  T* ptr_;
+  T space_[kInline];
+  uint32_t size_;
+  uint32_t capacity_;
+
+  void Init() {
+    ptr_ = space_;
+    size_ = 0;
+    capacity_ = kInline;
+  }
+
+  void Discard() {
+    if (ptr_ != space_) base_internal::LowLevelAlloc::Free(ptr_);
+  }
+
+  void Grow(uint32_t n) {
+    while (capacity_ < n) {
+      capacity_ *= 2;
+    }
+    size_t request = static_cast<size_t>(capacity_) * sizeof(T);
+    T* copy = static_cast<T*>(
+        base_internal::LowLevelAlloc::AllocWithArena(request, arena));
+    std::copy(ptr_, ptr_ + size_, copy);
+    Discard();
+    ptr_ = copy;
+  }
+
+  Vec(const Vec&) = delete;
+  Vec& operator=(const Vec&) = delete;
+};
+
+// A hash set of non-negative int32_t that uses Vec for its underlying storage.
+class NodeSet {
+ public:
+  NodeSet() { Init(); }
+
+  void clear() { Init(); }
+  bool contains(int32_t v) const { return table_[FindIndex(v)] == v; }
+
+  bool insert(int32_t v) {
+    uint32_t i = FindIndex(v);
+    if (table_[i] == v) {
+      return false;
+    }
+    if (table_[i] == kEmpty) {
+      // Only inserting over an empty cell increases the number of occupied
+      // slots.
+      occupied_++;
+    }
+    table_[i] = v;
+    // Double when 75% full.
+    if (occupied_ >= table_.size() - table_.size()/4) Grow();
+    return true;
+  }
+
+  void erase(uint32_t v) {
+    uint32_t i = FindIndex(v);
+    if (static_cast<uint32_t>(table_[i]) == v) {
+      table_[i] = kDel;
+    }
+  }
+
+  // Iteration: is done via HASH_FOR_EACH
+  // Example:
+  //    HASH_FOR_EACH(elem, node->out) { ... }
+#define HASH_FOR_EACH(elem, eset) \
+  for (int32_t elem, _cursor = 0; (eset).Next(&_cursor, &elem); )
+  bool Next(int32_t* cursor, int32_t* elem) {
+    while (static_cast<uint32_t>(*cursor) < table_.size()) {
+      int32_t v = table_[*cursor];
+      (*cursor)++;
+      if (v >= 0) {
+        *elem = v;
+        return true;
+      }
+    }
+    return false;
+  }
+
+ private:
+  enum : int32_t { kEmpty = -1, kDel = -2 };
+  Vec<int32_t> table_;
+  uint32_t occupied_;     // Count of non-empty slots (includes deleted slots)
+
+  static uint32_t Hash(uint32_t a) { return a * 41; }
+
+  // Return index for storing v.  May return an empty index or deleted index
+  int FindIndex(int32_t v) const {
+    // Search starting at hash index.
+    const uint32_t mask = table_.size() - 1;
+    uint32_t i = Hash(v) & mask;
+    int deleted_index = -1;  // If >= 0, index of first deleted element we see
+    while (true) {
+      int32_t e = table_[i];
+      if (v == e) {
+        return i;
+      } else if (e == kEmpty) {
+        // Return any previously encountered deleted slot.
+        return (deleted_index >= 0) ? deleted_index : i;
+      } else if (e == kDel && deleted_index < 0) {
+        // Keep searching since v might be present later.
+        deleted_index = i;
+      }
+      i = (i + 1) & mask;  // Linear probing; quadratic is slightly slower.
+    }
+  }
+
+  void Init() {
+    table_.clear();
+    table_.resize(kInline);
+    table_.fill(kEmpty);
+    occupied_ = 0;
+  }
+
+  void Grow() {
+    Vec<int32_t> copy;
+    copy.MoveFrom(&table_);
+    occupied_ = 0;
+    table_.resize(copy.size() * 2);
+    table_.fill(kEmpty);
+
+    for (const auto& e : copy) {
+      if (e >= 0) insert(e);
+    }
+  }
+
+  NodeSet(const NodeSet&) = delete;
+  NodeSet& operator=(const NodeSet&) = delete;
+};
+
+// We encode a node index and a node version in GraphId.  The version
+// number is incremented when the GraphId is freed which automatically
+// invalidates all copies of the GraphId.
+
+inline GraphId MakeId(int32_t index, uint32_t version) {
+  GraphId g;
+  g.handle =
+      (static_cast<uint64_t>(version) << 32) | static_cast<uint32_t>(index);
+  return g;
+}
+
+inline int32_t NodeIndex(GraphId id) {
+  return static_cast<uint32_t>(id.handle & 0xfffffffful);
+}
+
+inline uint32_t NodeVersion(GraphId id) {
+  return static_cast<uint32_t>(id.handle >> 32);
+}
+
+struct Node {
+  int32_t rank;               // rank number assigned by Pearce-Kelly algorithm
+  uint32_t version;           // Current version number
+  int32_t next_hash;          // Next entry in hash table
+  bool visited;               // Temporary marker used by depth-first-search
+  uintptr_t masked_ptr;       // User-supplied pointer
+  NodeSet in;                 // List of immediate predecessor nodes in graph
+  NodeSet out;                // List of immediate successor nodes in graph
+  int priority;               // Priority of recorded stack trace.
+  int nstack;                 // Depth of recorded stack trace.
+  void* stack[40];            // stack[0,nstack-1] holds stack trace for node.
+};
+
+// Hash table for pointer to node index lookups.
+class PointerMap {
+ public:
+  explicit PointerMap(const Vec<Node*>* nodes) : nodes_(nodes) {
+    table_.fill(-1);
+  }
+
+  int32_t Find(void* ptr) {
+    auto masked = base_internal::HidePtr(ptr);
+    for (int32_t i = table_[Hash(ptr)]; i != -1;) {
+      Node* n = (*nodes_)[i];
+      if (n->masked_ptr == masked) return i;
+      i = n->next_hash;
+    }
+    return -1;
+  }
+
+  void Add(void* ptr, int32_t i) {
+    int32_t* head = &table_[Hash(ptr)];
+    (*nodes_)[i]->next_hash = *head;
+    *head = i;
+  }
+
+  int32_t Remove(void* ptr) {
+    // Advance through linked list while keeping track of the
+    // predecessor slot that points to the current entry.
+    auto masked = base_internal::HidePtr(ptr);
+    for (int32_t* slot = &table_[Hash(ptr)]; *slot != -1; ) {
+      int32_t index = *slot;
+      Node* n = (*nodes_)[index];
+      if (n->masked_ptr == masked) {
+        *slot = n->next_hash;  // Remove n from linked list
+        n->next_hash = -1;
+        return index;
+      }
+      slot = &n->next_hash;
+    }
+    return -1;
+  }
+
+ private:
+  // Number of buckets in hash table for pointer lookups.
+  static constexpr uint32_t kHashTableSize = 8171;  // should be prime
+
+  const Vec<Node*>* nodes_;
+  std::array<int32_t, kHashTableSize> table_;
+
+  static uint32_t Hash(void* ptr) {
+    return reinterpret_cast<uintptr_t>(ptr) % kHashTableSize;
+  }
+};
+
+}  // namespace
+
+struct GraphCycles::Rep {
+  Vec<Node*> nodes_;
+  Vec<int32_t> free_nodes_;  // Indices for unused entries in nodes_
+  PointerMap ptrmap_;
+
+  // Temporary state.
+  Vec<int32_t> deltaf_;  // Results of forward DFS
+  Vec<int32_t> deltab_;  // Results of backward DFS
+  Vec<int32_t> list_;    // All nodes to reprocess
+  Vec<int32_t> merged_;  // Rank values to assign to list_ entries
+  Vec<int32_t> stack_;   // Emulates recursion stack for depth-first searches
+
+  Rep() : ptrmap_(&nodes_) {}
+};
+
+static Node* FindNode(GraphCycles::Rep* rep, GraphId id) {
+  Node* n = rep->nodes_[NodeIndex(id)];
+  return (n->version == NodeVersion(id)) ? n : nullptr;
+}
+
+GraphCycles::GraphCycles() {
+  InitArenaIfNecessary();
+  rep_ = new (base_internal::LowLevelAlloc::AllocWithArena(sizeof(Rep), arena))
+      Rep;
+}
+
+GraphCycles::~GraphCycles() {
+  for (auto* node : rep_->nodes_) {
+    node->Node::~Node();
+    base_internal::LowLevelAlloc::Free(node);
+  }
+  rep_->Rep::~Rep();
+  base_internal::LowLevelAlloc::Free(rep_);
+}
+
+bool GraphCycles::CheckInvariants() const {
+  Rep* r = rep_;
+  NodeSet ranks;  // Set of ranks seen so far.
+  for (uint32_t x = 0; x < r->nodes_.size(); x++) {
+    Node* nx = r->nodes_[x];
+    void* ptr = base_internal::UnhidePtr<void>(nx->masked_ptr);
+    if (ptr != nullptr && static_cast<uint32_t>(r->ptrmap_.Find(ptr)) != x) {
+      ABSL_RAW_LOG(FATAL, "Did not find live node in hash table %u %p", x, ptr);
+    }
+    if (nx->visited) {
+      ABSL_RAW_LOG(FATAL, "Did not clear visited marker on node %u", x);
+    }
+    if (!ranks.insert(nx->rank)) {
+      ABSL_RAW_LOG(FATAL, "Duplicate occurrence of rank %d", nx->rank);
+    }
+    HASH_FOR_EACH(y, nx->out) {
+      Node* ny = r->nodes_[y];
+      if (nx->rank >= ny->rank) {
+        ABSL_RAW_LOG(FATAL, "Edge %u->%d has bad rank assignment %d->%d", x, y,
+                     nx->rank, ny->rank);
+      }
+    }
+  }
+  return true;
+}
+
+GraphId GraphCycles::GetId(void* ptr) {
+  int32_t i = rep_->ptrmap_.Find(ptr);
+  if (i != -1) {
+    return MakeId(i, rep_->nodes_[i]->version);
+  } else if (rep_->free_nodes_.empty()) {
+    Node* n =
+        new (base_internal::LowLevelAlloc::AllocWithArena(sizeof(Node), arena))
+            Node;
+    n->version = 1;  // Avoid 0 since it is used by InvalidGraphId()
+    n->visited = false;
+    n->rank = rep_->nodes_.size();
+    n->masked_ptr = base_internal::HidePtr(ptr);
+    n->nstack = 0;
+    n->priority = 0;
+    rep_->nodes_.push_back(n);
+    rep_->ptrmap_.Add(ptr, n->rank);
+    return MakeId(n->rank, n->version);
+  } else {
+    // Preserve preceding rank since the set of ranks in use must be
+    // a permutation of [0,rep_->nodes_.size()-1].
+    int32_t r = rep_->free_nodes_.back();
+    rep_->free_nodes_.pop_back();
+    Node* n = rep_->nodes_[r];
+    n->masked_ptr = base_internal::HidePtr(ptr);
+    n->nstack = 0;
+    n->priority = 0;
+    rep_->ptrmap_.Add(ptr, r);
+    return MakeId(r, n->version);
+  }
+}
+
+void GraphCycles::RemoveNode(void* ptr) {
+  int32_t i = rep_->ptrmap_.Remove(ptr);
+  if (i == -1) {
+    return;
+  }
+  Node* x = rep_->nodes_[i];
+  HASH_FOR_EACH(y, x->out) {
+    rep_->nodes_[y]->in.erase(i);
+  }
+  HASH_FOR_EACH(y, x->in) {
+    rep_->nodes_[y]->out.erase(i);
+  }
+  x->in.clear();
+  x->out.clear();
+  x->masked_ptr = base_internal::HidePtr<void>(nullptr);
+  if (x->version == std::numeric_limits<uint32_t>::max()) {
+    // Cannot use x any more
+  } else {
+    x->version++;  // Invalidates all copies of node.
+    rep_->free_nodes_.push_back(i);
+  }
+}
+
+void* GraphCycles::Ptr(GraphId id) {
+  Node* n = FindNode(rep_, id);
+  return n == nullptr ? nullptr
+                      : base_internal::UnhidePtr<void>(n->masked_ptr);
+}
+
+bool GraphCycles::HasNode(GraphId node) {
+  return FindNode(rep_, node) != nullptr;
+}
+
+bool GraphCycles::HasEdge(GraphId x, GraphId y) const {
+  Node* xn = FindNode(rep_, x);
+  return xn && FindNode(rep_, y) && xn->out.contains(NodeIndex(y));
+}
+
+void GraphCycles::RemoveEdge(GraphId x, GraphId y) {
+  Node* xn = FindNode(rep_, x);
+  Node* yn = FindNode(rep_, y);
+  if (xn && yn) {
+    xn->out.erase(NodeIndex(y));
+    yn->in.erase(NodeIndex(x));
+    // No need to update the rank assignment since a previous valid
+    // rank assignment remains valid after an edge deletion.
+  }
+}
+
+static bool ForwardDFS(GraphCycles::Rep* r, int32_t n, int32_t upper_bound);
+static void BackwardDFS(GraphCycles::Rep* r, int32_t n, int32_t lower_bound);
+static void Reorder(GraphCycles::Rep* r);
+static void Sort(const Vec<Node*>&, Vec<int32_t>* delta);
+static void MoveToList(
+    GraphCycles::Rep* r, Vec<int32_t>* src, Vec<int32_t>* dst);
+
+bool GraphCycles::InsertEdge(GraphId idx, GraphId idy) {
+  Rep* r = rep_;
+  const int32_t x = NodeIndex(idx);
+  const int32_t y = NodeIndex(idy);
+  Node* nx = FindNode(r, idx);
+  Node* ny = FindNode(r, idy);
+  if (nx == nullptr || ny == nullptr) return true;  // Expired ids
+
+  if (nx == ny) return false;  // Self edge
+  if (!nx->out.insert(y)) {
+    // Edge already exists.
+    return true;
+  }
+
+  ny->in.insert(x);
+
+  if (nx->rank <= ny->rank) {
+    // New edge is consistent with existing rank assignment.
+    return true;
+  }
+
+  // Current rank assignments are incompatible with the new edge.  Recompute.
+  // We only need to consider nodes that fall in the range [ny->rank,nx->rank].
+  if (!ForwardDFS(r, y, nx->rank)) {
+    // Found a cycle.  Undo the insertion and tell caller.
+    nx->out.erase(y);
+    ny->in.erase(x);
+    // Since we do not call Reorder() on this path, clear any visited
+    // markers left by ForwardDFS.
+    for (const auto& d : r->deltaf_) {
+      r->nodes_[d]->visited = false;
+    }
+    return false;
+  }
+  BackwardDFS(r, x, ny->rank);
+  Reorder(r);
+  return true;
+}
+
+static bool ForwardDFS(GraphCycles::Rep* r, int32_t n, int32_t upper_bound) {
+  // Avoid recursion since stack space might be limited.
+  // We instead keep a stack of nodes to visit.
+  r->deltaf_.clear();
+  r->stack_.clear();
+  r->stack_.push_back(n);
+  while (!r->stack_.empty()) {
+    n = r->stack_.back();
+    r->stack_.pop_back();
+    Node* nn = r->nodes_[n];
+    if (nn->visited) continue;
+
+    nn->visited = true;
+    r->deltaf_.push_back(n);
+
+    HASH_FOR_EACH(w, nn->out) {
+      Node* nw = r->nodes_[w];
+      if (nw->rank == upper_bound) {
+        return false;  // Cycle
+      }
+      if (!nw->visited && nw->rank < upper_bound) {
+        r->stack_.push_back(w);
+      }
+    }
+  }
+  return true;
+}
+
+static void BackwardDFS(GraphCycles::Rep* r, int32_t n, int32_t lower_bound) {
+  r->deltab_.clear();
+  r->stack_.clear();
+  r->stack_.push_back(n);
+  while (!r->stack_.empty()) {
+    n = r->stack_.back();
+    r->stack_.pop_back();
+    Node* nn = r->nodes_[n];
+    if (nn->visited) continue;
+
+    nn->visited = true;
+    r->deltab_.push_back(n);
+
+    HASH_FOR_EACH(w, nn->in) {
+      Node* nw = r->nodes_[w];
+      if (!nw->visited && lower_bound < nw->rank) {
+        r->stack_.push_back(w);
+      }
+    }
+  }
+}
+
+static void Reorder(GraphCycles::Rep* r) {
+  Sort(r->nodes_, &r->deltab_);
+  Sort(r->nodes_, &r->deltaf_);
+
+  // Adds contents of delta lists to list_ (backwards deltas first).
+  r->list_.clear();
+  MoveToList(r, &r->deltab_, &r->list_);
+  MoveToList(r, &r->deltaf_, &r->list_);
+
+  // Produce sorted list of all ranks that will be reassigned.
+  r->merged_.resize(r->deltab_.size() + r->deltaf_.size());
+  std::merge(r->deltab_.begin(), r->deltab_.end(),
+             r->deltaf_.begin(), r->deltaf_.end(),
+             r->merged_.begin());
+
+  // Assign the ranks in order to the collected list.
+  for (uint32_t i = 0; i < r->list_.size(); i++) {
+    r->nodes_[r->list_[i]]->rank = r->merged_[i];
+  }
+}
+
+static void Sort(const Vec<Node*>& nodes, Vec<int32_t>* delta) {
+  struct ByRank {
+    const Vec<Node*>* nodes;
+    bool operator()(int32_t a, int32_t b) const {
+      return (*nodes)[a]->rank < (*nodes)[b]->rank;
+    }
+  };
+  ByRank cmp;
+  cmp.nodes = &nodes;
+  std::sort(delta->begin(), delta->end(), cmp);
+}
+
+static void MoveToList(
+    GraphCycles::Rep* r, Vec<int32_t>* src, Vec<int32_t>* dst) {
+  for (auto& v : *src) {
+    int32_t w = v;
+    v = r->nodes_[w]->rank;         // Replace v entry with its rank
+    r->nodes_[w]->visited = false;  // Prepare for future DFS calls
+    dst->push_back(w);
+  }
+}
+
+int GraphCycles::FindPath(GraphId idx, GraphId idy, int max_path_len,
+                          GraphId path[]) const {
+  Rep* r = rep_;
+  if (FindNode(r, idx) == nullptr || FindNode(r, idy) == nullptr) return 0;
+  const int32_t x = NodeIndex(idx);
+  const int32_t y = NodeIndex(idy);
+
+  // Forward depth first search starting at x until we hit y.
+  // As we descend into a node, we push it onto the path.
+  // As we leave a node, we remove it from the path.
+  int path_len = 0;
+
+  NodeSet seen;
+  r->stack_.clear();
+  r->stack_.push_back(x);
+  while (!r->stack_.empty()) {
+    int32_t n = r->stack_.back();
+    r->stack_.pop_back();
+    if (n < 0) {
+      // Marker to indicate that we are leaving a node
+      path_len--;
+      continue;
+    }
+
+    if (path_len < max_path_len) {
+      path[path_len] = MakeId(n, rep_->nodes_[n]->version);
+    }
+    path_len++;
+    r->stack_.push_back(-1);  // Will remove tentative path entry
+
+    if (n == y) {
+      return path_len;
+    }
+
+    HASH_FOR_EACH(w, r->nodes_[n]->out) {
+      if (seen.insert(w)) {
+        r->stack_.push_back(w);
+      }
+    }
+  }
+
+  return 0;
+}
+
+bool GraphCycles::IsReachable(GraphId x, GraphId y) const {
+  return FindPath(x, y, 0, nullptr) > 0;
+}
+
+void GraphCycles::UpdateStackTrace(GraphId id, int priority,
+                                   int (*get_stack_trace)(void** stack, int)) {
+  Node* n = FindNode(rep_, id);
+  if (n == nullptr || n->priority >= priority) {
+    return;
+  }
+  n->nstack = (*get_stack_trace)(n->stack, ABSL_ARRAYSIZE(n->stack));
+  n->priority = priority;
+}
+
+int GraphCycles::GetStackTrace(GraphId id, void*** ptr) {
+  Node* n = FindNode(rep_, id);
+  if (n == nullptr) {
+    *ptr = nullptr;
+    return 0;
+  } else {
+    *ptr = n->stack;
+    return n->nstack;
+  }
+}
+
+}  // namespace synchronization_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_LOW_LEVEL_ALLOC_MISSING