grpc 1.46.3-x86_64-linux → 1.48.0-x86_64-linux

data/src/core/lib/event_engine/iomgr_engine/timer.h

@@ -0,0 +1,193 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIMER_H
+#define GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIMER_H
+
+#include <grpc/support/port_platform.h>
+
+#include <stddef.h>
+
+#include <atomic>
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/types/optional.h"
+
+#include <grpc/event_engine/event_engine.h>
+
+#include "src/core/lib/event_engine/iomgr_engine/time_averaged_stats.h"
+#include "src/core/lib/event_engine/iomgr_engine/timer_heap.h"
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/gprpp/time.h"
+
+namespace grpc_event_engine {
+namespace iomgr_engine {
+
+struct Timer {
+  int64_t deadline;
+  // kInvalidHeapIndex if not in heap.
+  size_t heap_index;
+  bool pending;
+  struct Timer* next;
+  struct Timer* prev;
+  experimental::EventEngine::Closure* closure;
+#ifndef NDEBUG
+  struct Timer* hash_table_next;
+#endif
+
+  grpc_event_engine::experimental::EventEngine::TaskHandle task_handle;
+};
+
+// Dependency injection: allow tests and/or TimerManager to inject
+// their own implementations of Now, Kick.
+class TimerListHost {
+ public:
+  // Return the current timestamp.
+  // Abstracted so that tests can be run deterministically.
+  virtual grpc_core::Timestamp Now() = 0;
+  // Wake up a thread to check for timers.
+  virtual void Kick() = 0;
+
+ protected:
+  ~TimerListHost() = default;
+};
+
+class TimerList {
+ public:
+  explicit TimerList(TimerListHost* host);
+
+  TimerList(const TimerList&) = delete;
+  TimerList& operator=(const TimerList&) = delete;
+
+  /* Initialize *timer. When expired or canceled, closure will be called with
+   error set to indicate if it expired (GRPC_ERROR_NONE) or was canceled
+   (GRPC_ERROR_CANCELLED). *closure is guaranteed to be called exactly once, and
+   application code should check the error to determine how it was invoked. The
+   application callback is also responsible for maintaining information about
+   when to free up any user-level state. Behavior is undefined for a deadline of
+   grpc_core::Timestamp::InfFuture(). */
+  void TimerInit(Timer* timer, grpc_core::Timestamp deadline,
+                 experimental::EventEngine::Closure* closure);
+
+  /* Note that there is no timer destroy function. This is because the
+     timer is a one-time occurrence with a guarantee that the callback will
+     be called exactly once, either at expiration or cancellation. Thus, all
+     the internal timer event management state is destroyed just before
+     that callback is invoked. If the user has additional state associated with
+     the timer, the user is responsible for determining when it is safe to
+     destroy that state. */
+
+  /* Cancel an *timer.
+     There are three cases:
+     1. We normally cancel the timer
+     2. The timer has already run
+     3. We can't cancel the timer because it is "in flight".
+
+     In all of these cases, the cancellation is still considered successful.
+     They are essentially distinguished in that the timer_cb will be run
+     exactly once from either the cancellation (with error GRPC_ERROR_CANCELLED)
+     or from the activation (with error GRPC_ERROR_NONE).
+
+     Note carefully that the callback function MAY occur in the same callstack
+     as grpc_timer_cancel. It's expected that most timers will be cancelled
+     (their primary use is to implement deadlines), and so this code is
+     optimized such that cancellation costs as little as possible. Making
+     callbacks run inline matches this aim.
+
+     Requires: cancel() must happen after init() on a given timer */
+  bool TimerCancel(Timer* timer) GRPC_MUST_USE_RESULT;
+
+  /* iomgr internal api for dealing with timers */
+
+  /* Check for timers to be run, and return them.
+     Return nullopt if timers could not be checked due to contention with
+     another thread checking.
+     Return a vector of closures that *must* be run otherwise.
+     If next is non-null, TRY to update *next with the next running timer
+     IF that timer occurs before *next current value.
+     *next is never guaranteed to be updated on any given execution; however,
+     with high probability at least one thread in the system will see an update
+     at any time slice. */
+  absl::optional<std::vector<experimental::EventEngine::Closure*>> TimerCheck(
+      grpc_core::Timestamp* next);
+
+ private:
+  /* A "timer shard". Contains a 'heap' and a 'list' of timers. All timers with
+   * deadlines earlier than 'queue_deadline_cap' are maintained in the heap and
+   * others are maintained in the list (unordered). This helps to keep the
+   * number of elements in the heap low.
+   *
+   * The 'queue_deadline_cap' gets recomputed periodically based on the timer
+   * stats maintained in 'stats' and the relevant timers are then moved from the
+   * 'list' to 'heap'.
+   */
+  struct Shard {
+    Shard();
+
+    grpc_core::Timestamp ComputeMinDeadline() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu);
+    bool RefillHeap(grpc_core::Timestamp now) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu);
+    Timer* PopOne(grpc_core::Timestamp now) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu);
+    void PopTimers(grpc_core::Timestamp now,
+                   grpc_core::Timestamp* new_min_deadline,
+                   std::vector<experimental::EventEngine::Closure*>* out)
+        ABSL_LOCKS_EXCLUDED(mu);
+
+    grpc_core::Mutex mu;
+    TimeAveragedStats stats ABSL_GUARDED_BY(mu);
+    /* All and only timers with deadlines < this will be in the heap. */
+    grpc_core::Timestamp queue_deadline_cap ABSL_GUARDED_BY(mu);
+    /* The deadline of the next timer due in this shard. */
+    grpc_core::Timestamp min_deadline ABSL_GUARDED_BY(&TimerList::mu_);
+    /* Index of this timer_shard in the g_shard_queue. */
+    uint32_t shard_queue_index ABSL_GUARDED_BY(&TimerList::mu_);
+    /* This holds all timers with deadlines < queue_deadline_cap. Timers in this
+       list have the top bit of their deadline set to 0. */
+    TimerHeap heap ABSL_GUARDED_BY(mu);
+    /* This holds timers whose deadline is >= queue_deadline_cap. */
+    Timer list ABSL_GUARDED_BY(mu);
+  };
+
+  void SwapAdjacentShardsInQueue(uint32_t first_shard_queue_index)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+  void NoteDeadlineChange(Shard* shard) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+  std::vector<experimental::EventEngine::Closure*> FindExpiredTimers(
+      grpc_core::Timestamp now, grpc_core::Timestamp* next);
+
+  TimerListHost* const host_;
+  const size_t num_shards_;
+  grpc_core::Mutex mu_;
+  /* The deadline of the next timer due across all timer shards */
+  std::atomic<uint64_t> min_timer_;
+  /* Allow only one FindExpiredTimers at once (used as a TryLock, protects no
+   * fields but ensures limits on concurrency) */
+  grpc_core::Mutex checker_mu_;
+  /* Array of timer shards. Whenever a timer (Timer *) is added, its address
+   * is hashed to select the timer shard to add the timer to */
+  const std::unique_ptr<Shard[]> shards_;
+  /* Maintains a sorted list of timer shards (sorted by their min_deadline, i.e
+   * the deadline of the next timer in each shard). */
+  const std::unique_ptr<Shard*[]> shard_queue_ ABSL_GUARDED_BY(mu_);
+};
+
+}  // namespace iomgr_engine
+}  // namespace grpc_event_engine
+
+#endif /* GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIMER_H */

data/src/core/lib/event_engine/iomgr_engine/timer_heap.cc

@@ -0,0 +1,107 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/event_engine/iomgr_engine/timer_heap.h"
+
+#include <stdint.h>
+
+#include <algorithm>
+
+#include "src/core/lib/event_engine/iomgr_engine/timer.h"
+
+namespace grpc_event_engine {
+namespace iomgr_engine {
+
+/* Adjusts a heap so as to move a hole at position i closer to the root,
+   until a suitable position is found for element t. Then, copies t into that
+   position. This functor is called each time immediately after modifying a
+   value in the underlying container, with the offset of the modified element as
+   its argument. */
+void TimerHeap::AdjustUpwards(size_t i, Timer* t) {
+  while (i > 0) {
+    size_t parent = (i - 1) / 2;
+    if (timers_[parent]->deadline <= t->deadline) break;
+    timers_[i] = timers_[parent];
+    timers_[i]->heap_index = i;
+    i = parent;
+  }
+  timers_[i] = t;
+  t->heap_index = i;
+}
+
+/* Adjusts a heap so as to move a hole at position i farther away from the root,
+   until a suitable position is found for element t.  Then, copies t into that
+   position. */
+void TimerHeap::AdjustDownwards(size_t i, Timer* t) {
+  for (;;) {
+    size_t left_child = 1 + 2 * i;
+    if (left_child >= timers_.size()) break;
+    size_t right_child = left_child + 1;
+    size_t next_i =
+        right_child < timers_.size() &&
+                timers_[left_child]->deadline > timers_[right_child]->deadline
+            ? right_child
+            : left_child;
+    if (t->deadline <= timers_[next_i]->deadline) break;
+    timers_[i] = timers_[next_i];
+    timers_[i]->heap_index = i;
+    i = next_i;
+  }
+  timers_[i] = t;
+  t->heap_index = i;
+}
+
+void TimerHeap::NoteChangedPriority(Timer* timer) {
+  uint32_t i = timer->heap_index;
+  uint32_t parent = static_cast<uint32_t>((static_cast<int>(i) - 1) / 2);
+  if (timers_[parent]->deadline > timer->deadline) {
+    AdjustUpwards(i, timer);
+  } else {
+    AdjustDownwards(i, timer);
+  }
+}
+
+bool TimerHeap::Add(Timer* timer) {
+  timer->heap_index = timers_.size();
+  timers_.push_back(timer);
+  AdjustUpwards(timer->heap_index, timer);
+  return timer->heap_index == 0;
+}
+
+void TimerHeap::Remove(Timer* timer) {
+  uint32_t i = timer->heap_index;
+  if (i == timers_.size() - 1) {
+    timers_.pop_back();
+    return;
+  }
+  timers_[i] = timers_[timers_.size() - 1];
+  timers_[i]->heap_index = i;
+  timers_.pop_back();
+  NoteChangedPriority(timers_[i]);
+}
+
+bool TimerHeap::is_empty() { return timers_.empty(); }
+
+Timer* TimerHeap::Top() { return timers_[0]; }
+
+void TimerHeap::Pop() { Remove(Top()); }
+
+}  // namespace iomgr_engine
+}  // namespace grpc_event_engine

data/src/core/lib/event_engine/iomgr_engine/timer_heap.h

@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIMER_HEAP_H
+#define GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIMER_HEAP_H
+
+#include <grpc/support/port_platform.h>
+
+#include <cstddef>
+#include <vector>
+
+namespace grpc_event_engine {
+namespace iomgr_engine {
+
+struct Timer;
+
+class TimerHeap {
+ public:
+  /* return true if the new timer is the first timer in the heap */
+  bool Add(Timer* timer);
+
+  void Remove(Timer* timer);
+  Timer* Top();
+  void Pop();
+
+  bool is_empty();
+
+  const std::vector<Timer*>& TestOnlyGetTimers() const { return timers_; }
+
+ private:
+  void AdjustUpwards(size_t i, Timer* t);
+  void AdjustDownwards(size_t i, Timer* t);
+  void NoteChangedPriority(Timer* timer);
+
+  std::vector<Timer*> timers_;
+};
+
+}  // namespace iomgr_engine
+}  // namespace grpc_event_engine
+
+#endif /* GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIMER_HEAP_H */

data/src/core/lib/event_engine/iomgr_engine/timer_manager.cc

@@ -0,0 +1,254 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/event_engine/iomgr_engine/timer_manager.h"
+
+#include <algorithm>
+#include <memory>
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "absl/time/time.h"
+#include "absl/types/optional.h"
+
+#include <grpc/impl/codegen/gpr_types.h>
+#include <grpc/support/log.h>
+#include <grpc/support/time.h>
+
+#include "src/core/lib/gprpp/thd.h"
+
+namespace grpc_event_engine {
+namespace iomgr_engine {
+
+namespace {
+class ThreadCollector {
+ public:
+  ThreadCollector() = default;
+  ~ThreadCollector();
+
+  void Collect(std::vector<grpc_core::Thread> threads) {
+    GPR_ASSERT(threads_.empty());
+    threads_ = std::move(threads);
+  }
+
+ private:
+  std::vector<grpc_core::Thread> threads_;
+};
+
+ThreadCollector::~ThreadCollector() {
+  for (auto& t : threads_) t.Join();
+}
+}  // namespace
+
+void TimerManager::StartThread() {
+  ++waiter_count_;
+  ++thread_count_;
+  auto* thread = new RunThreadArgs();
+  thread->self = this;
+  thread->thread =
+      grpc_core::Thread("timer_manager", &TimerManager::RunThread, thread);
+  thread->thread.Start();
+}
+
+void TimerManager::RunSomeTimers(
+    std::vector<experimental::EventEngine::Closure*> timers) {
+  // if there's something to execute...
+  ThreadCollector collector;
+  {
+    grpc_core::MutexLock lock(&mu_);
+    // remove a waiter from the pool, and start another thread if necessary
+    --waiter_count_;
+    if (waiter_count_ == 0) {
+      // The number of timer threads is always increasing until all the threads
+      // are stopped. In rare cases, if a large number of timers fire
+      // simultaneously, we may end up using a large number of threads.
+      // TODO(ctiller): We could avoid this by exiting threads in WaitUntil().
+      StartThread();
+    } else {
+      // if there's no thread waiting with a timeout, kick an existing untimed
+      // waiter so that the next deadline is not missed
+      if (!has_timed_waiter_) {
+        cv_.Signal();
+      }
+    }
+  }
+  for (auto* timer : timers) {
+    timer->Run();
+  }
+  {
+    grpc_core::MutexLock lock(&mu_);
+    collector.Collect(std::move(completed_threads_));
+    // get ready to wait again
+    ++waiter_count_;
+  }
+}
+
+// wait until 'next' (or forever if there is already a timed waiter in the pool)
+// returns true if the thread should continue executing (false if it should
+// shutdown)
+bool TimerManager::WaitUntil(grpc_core::Timestamp next) {
+  grpc_core::MutexLock lock(&mu_);
+
+  if (shutdown_) {
+    return false;
+  }
+
+  // TODO(ctiller): if there are too many waiting threads, this would be a good
+  // place to exit the current thread.
+
+  // If kicked_ is true at this point, it means there was a kick from the timer
+  // system that the timer-manager threads here missed. We cannot trust 'next'
+  // here any longer (since there might be an earlier deadline). So if kicked_
+  // is true at this point, we should quickly exit this and get the next
+  // deadline from the timer system
+
+  if (!kicked_) {
+    // if there's no timed waiter, we should become one: that waiter waits
+    // only until the next timer should expire. All other timers wait forever
+    //
+    // 'timed_waiter_generation_' is a global generation counter. The idea here
+    // is that the thread becoming a timed-waiter increments and stores this
+    // global counter locally in 'my_timed_waiter_generation' before going to
+    // sleep. After waking up, if my_timed_waiter_generation ==
+    // timed_waiter_generation_, it can be sure that it was the timed_waiter
+    // thread (and that no other thread took over while this was asleep)
+    //
+    // Initialize my_timed_waiter_generation to some value that is NOT equal to
+    // timed_waiter_generation_
+    uint64_t my_timed_waiter_generation = timed_waiter_generation_ - 1;
+
+    /* If there's no timed waiter, we should become one: that waiter waits only
+       until the next timer should expire. All other timer threads wait forever
+       unless their 'next' is earlier than the current timed-waiter's deadline
+       (in which case the thread with earlier 'next' takes over as the new timed
+       waiter) */
+    if (next != grpc_core::Timestamp::InfFuture()) {
+      if (!has_timed_waiter_ || (next < timed_waiter_deadline_)) {
+        my_timed_waiter_generation = ++timed_waiter_generation_;
+        has_timed_waiter_ = true;
+        timed_waiter_deadline_ = next;
+      } else {  // timed_waiter_ == true && next >= timed_waiter_deadline_
+        next = grpc_core::Timestamp::InfFuture();
+      }
+    }
+
+    cv_.WaitWithTimeout(&mu_,
+                        absl::Milliseconds((next - host_.Now()).millis()));
+
+    // if this was the timed waiter, then we need to check timers, and flag
+    // that there's now no timed waiter... we'll look for a replacement if
+    // there's work to do after checking timers (code above)
+    if (my_timed_waiter_generation == timed_waiter_generation_) {
+      ++wakeups_;
+      has_timed_waiter_ = false;
+      timed_waiter_deadline_ = grpc_core::Timestamp::InfFuture();
+    }
+  }
+
+  kicked_ = false;
+
+  return true;
+}
+
+void TimerManager::MainLoop() {
+  for (;;) {
+    grpc_core::Timestamp next = grpc_core::Timestamp::InfFuture();
+    absl::optional<std::vector<experimental::EventEngine::Closure*>>
+        check_result = timer_list_->TimerCheck(&next);
+    if (check_result.has_value()) {
+      if (!check_result->empty()) {
+        RunSomeTimers(std::move(*check_result));
+        continue;
+      }
+    } else {
+      /* This case only happens under contention, meaning more than one timer
+         manager thread checked timers concurrently.
+
+         If that happens, we're guaranteed that some other thread has just
+         checked timers, and this will avalanche into some other thread seeing
+         empty timers and doing a timed sleep.
+
+         Consequently, we can just sleep forever here and be happy at some
+         saved wakeup cycles. */
+      next = grpc_core::Timestamp::InfFuture();
+    }
+    if (!WaitUntil(next)) return;
+  }
+}
+
+void TimerManager::RunThread(void* arg) {
+  std::unique_ptr<RunThreadArgs> thread(static_cast<RunThreadArgs*>(arg));
+  thread->self->MainLoop();
+  {
+    grpc_core::MutexLock lock(&thread->self->mu_);
+    thread->self->thread_count_--;
+    thread->self->completed_threads_.push_back(std::move(thread->thread));
+  }
+  thread->self->cv_.Signal();
+}
+
+TimerManager::TimerManager() : host_(this) {
+  timer_list_ = absl::make_unique<TimerList>(&host_);
+  grpc_core::MutexLock lock(&mu_);
+  StartThread();
+}
+
+grpc_core::Timestamp TimerManager::Host::Now() {
+  return grpc_core::Timestamp::FromTimespecRoundDown(
+      gpr_now(GPR_CLOCK_MONOTONIC));
+}
+
+void TimerManager::TimerInit(Timer* timer, grpc_core::Timestamp deadline,
+                             experimental::EventEngine::Closure* closure) {
+  timer_list_->TimerInit(timer, deadline, closure);
+}
+
+bool TimerManager::TimerCancel(Timer* timer) {
+  return timer_list_->TimerCancel(timer);
+}
+
+TimerManager::~TimerManager() {
+  {
+    grpc_core::MutexLock lock(&mu_);
+    shutdown_ = true;
+    cv_.SignalAll();
+  }
+  while (true) {
+    ThreadCollector collector;
+    grpc_core::MutexLock lock(&mu_);
+    collector.Collect(std::move(completed_threads_));
+    if (thread_count_ == 0) break;
+    cv_.Wait(&mu_);
+  }
+}
+
+void TimerManager::Host::Kick() { timer_manager_->Kick(); }
+
+void TimerManager::Kick() {
+  grpc_core::MutexLock lock(&mu_);
+  has_timed_waiter_ = false;
+  timed_waiter_deadline_ = grpc_core::Timestamp::InfFuture();
+  ++timed_waiter_generation_;
+  kicked_ = true;
+  cv_.Signal();
+}
+
+}  // namespace iomgr_engine
+}  // namespace grpc_event_engine