grpc 1.46.3-x86_64-linux → 1.48.0-x86_64-linux

data/src/core/lib/event_engine/iomgr_engine/thread_pool.h

@@ -0,0 +1,70 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_THREAD_POOL_H
+#define GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_THREAD_POOL_H
+
+#include <grpc/support/port_platform.h>
+
+#include <functional>
+#include <queue>
+#include <vector>
+
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/gprpp/thd.h"
+
+namespace grpc_event_engine {
+namespace iomgr_engine {
+
+class ThreadPool final {
+ public:
+  explicit ThreadPool(int reserve_threads);
+  ~ThreadPool();
+
+  void Add(const std::function<void()>& callback);
+
+ private:
+  class Thread {
+   public:
+    explicit Thread(ThreadPool* pool);
+    ~Thread();
+
+   private:
+    ThreadPool* pool_;
+    grpc_core::Thread thd_;
+    void ThreadFunc();
+  };
+
+  void ThreadFunc();
+  static void ReapThreads(std::vector<Thread*>* tlist);
+
+  grpc_core::Mutex mu_;
+  grpc_core::CondVar cv_;
+  grpc_core::CondVar shutdown_cv_;
+  bool shutdown_;
+  std::queue<std::function<void()>> callbacks_;
+  int reserve_threads_;
+  int nthreads_;
+  int threads_waiting_;
+  std::vector<Thread*> dead_threads_;
+};
+
+}  // namespace iomgr_engine
+}  // namespace grpc_event_engine
+
+#endif  // GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_THREAD_POOL_H

data/src/core/lib/event_engine/iomgr_engine/time_averaged_stats.cc

@@ -0,0 +1,62 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/event_engine/iomgr_engine/time_averaged_stats.h"
+
+namespace grpc_event_engine {
+namespace iomgr_engine {
+
+TimeAveragedStats::TimeAveragedStats(double init_avg, double regress_weight,
+                                     double persistence_factor)
+    : init_avg_(init_avg),
+      regress_weight_(regress_weight),
+      persistence_factor_(persistence_factor) {}
+
+void TimeAveragedStats::AddSample(double value) {
+  batch_total_value_ += value;
+  ++batch_num_samples_;
+}
+
+double TimeAveragedStats::UpdateAverage() {
+  /* Start with the current batch: */
+  double weighted_sum = batch_total_value_;
+  double total_weight = batch_num_samples_;
+  if (regress_weight_ > 0) {
+    /* Add in the regression towards init_avg_: */
+    weighted_sum += regress_weight_ * init_avg_;
+    total_weight += regress_weight_;
+  }
+  if (persistence_factor_ > 0) {
+    /* Add in the persistence: */
+    const double prev_sample_weight =
+        persistence_factor_ * aggregate_total_weight_;
+    weighted_sum += prev_sample_weight * aggregate_weighted_avg_;
+    total_weight += prev_sample_weight;
+  }
+  aggregate_weighted_avg_ =
+      (total_weight > 0) ? (weighted_sum / total_weight) : init_avg_;
+  aggregate_total_weight_ = total_weight;
+  batch_num_samples_ = 0;
+  batch_total_value_ = 0;
+  return aggregate_weighted_avg_;
+}
+
+}  // namespace iomgr_engine
+}  // namespace grpc_event_engine

data/src/core/lib/event_engine/iomgr_engine/time_averaged_stats.h

@@ -0,0 +1,81 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIME_AVERAGED_STATS_H
+#define GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIME_AVERAGED_STATS_H
+
+namespace grpc_event_engine {
+namespace iomgr_engine {
+
+/* This tracks a time-decaying weighted average.  It works by collecting
+   batches of samples and then mixing their average into a time-decaying
+   weighted mean.  It is designed for batch operations where we do many adds
+   before updating the average. */
+
+class TimeAveragedStats {
+ public:
+  TimeAveragedStats(double init_avg, double regress_weight,
+                    double persistence_factor);
+
+  /* Add a sample to the current batch. */
+  void AddSample(double value);
+  /* Complete a batch and compute the new estimate of the average sample
+     value. */
+  double UpdateAverage();
+
+  double aggregate_weighted_avg() const { return aggregate_weighted_avg_; }
+  double aggregate_total_weight() const { return aggregate_total_weight_; }
+
+ private:
+  /* The initial average value.  This is the reported average until the first
+     grpc_time_averaged_stats_update_average call.  If a positive regress_weight
+     is used, we also regress towards this value on each update. */
+  const double init_avg_;
+  /* The sample weight of "init_avg" that is mixed in with each call to
+     grpc_time_averaged_stats_update_average.  If the calls to
+     grpc_time_averaged_stats_add_sample stop, this will cause the average to
+     regress back to the mean.  This should be non-negative.  Set it to 0 to
+     disable the bias.  A value of 1 has the effect of adding in 1 bonus sample
+     with value init_avg to each sample period. */
+  const double regress_weight_;
+  /* This determines the rate of decay of the time-averaging from one period
+     to the next by scaling the aggregate_total_weight of samples from prior
+     periods when combining with the latest period.  It should be in the range
+     [0,1].  A higher value adapts more slowly.  With a value of 0.5, if the
+     batches each have k samples, the samples_in_avg_ will grow to 2 k, so the
+     weighting of the time average will eventually be 1/3 new batch and 2/3
+     old average. */
+  const double persistence_factor_;
+
+  /* The total value of samples since the last UpdateAverage(). */
+  double batch_total_value_ = 0;
+  /* The number of samples since the last UpdateAverage(). */
+  double batch_num_samples_ = 0;
+  /* The time-decayed sum of batch_num_samples_ over previous batches.  This is
+     the "weight" of the old aggregate_weighted_avg_ when updating the
+     average. */
+  double aggregate_total_weight_ = 0;
+  /* A time-decayed average of the (batch_total_value_ / batch_num_samples_),
+     computed by decaying the samples_in_avg_ weight in the weighted average. */
+  double aggregate_weighted_avg_ = init_avg_;
+};
+
+}  // namespace iomgr_engine
+}  // namespace grpc_event_engine
+
+#endif /* GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIME_AVERAGED_STATS_H */

data/src/core/lib/event_engine/iomgr_engine/timer.cc

@@ -0,0 +1,312 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/event_engine/iomgr_engine/timer.h"
+
+#include <algorithm>
+#include <atomic>
+#include <limits>
+#include <type_traits>
+#include <utility>
+
+#include <grpc/support/cpu.h>
+
+#include "src/core/lib/event_engine/iomgr_engine/timer_heap.h"
+#include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/gprpp/time.h"
+
+namespace grpc_event_engine {
+namespace iomgr_engine {
+
+static const size_t kInvalidHeapIndex = std::numeric_limits<size_t>::max();
+static const double kAddDeadlineScale = 0.33;
+static const double kMinQueueWindowDuration = 0.01;
+static const double kMaxQueueWindowDuration = 1.0;
+
+grpc_core::Timestamp TimerList::Shard::ComputeMinDeadline() {
+  return heap.is_empty()
+             ? queue_deadline_cap + grpc_core::Duration::Epsilon()
+             : grpc_core::Timestamp::FromMillisecondsAfterProcessEpoch(
+                   heap.Top()->deadline);
+}
+
+TimerList::Shard::Shard() : stats(1.0 / kAddDeadlineScale, 0.1, 0.5) {}
+
+TimerList::TimerList(TimerListHost* host)
+    : host_(host),
+      num_shards_(grpc_core::Clamp(2 * gpr_cpu_num_cores(), 1u, 32u)),
+      min_timer_(host_->Now().milliseconds_after_process_epoch()),
+      shards_(new Shard[num_shards_]),
+      shard_queue_(new Shard*[num_shards_]) {
+  for (size_t i = 0; i < num_shards_; i++) {
+    Shard& shard = shards_[i];
+    shard.queue_deadline_cap =
+        grpc_core::Timestamp::FromMillisecondsAfterProcessEpoch(
+            min_timer_.load(std::memory_order_relaxed));
+    shard.shard_queue_index = i;
+    shard.list.next = shard.list.prev = &shard.list;
+    shard.min_deadline = shard.ComputeMinDeadline();
+    shard_queue_[i] = &shard;
+  }
+}
+
+namespace {
+/* returns true if the first element in the list */
+void ListJoin(Timer* head, Timer* timer) {
+  timer->next = head;
+  timer->prev = head->prev;
+  timer->next->prev = timer->prev->next = timer;
+}
+
+void ListRemove(Timer* timer) {
+  timer->next->prev = timer->prev;
+  timer->prev->next = timer->next;
+}
+}  // namespace
+
+void TimerList::SwapAdjacentShardsInQueue(uint32_t first_shard_queue_index) {
+  Shard* temp;
+  temp = shard_queue_[first_shard_queue_index];
+  shard_queue_[first_shard_queue_index] =
+      shard_queue_[first_shard_queue_index + 1];
+  shard_queue_[first_shard_queue_index + 1] = temp;
+  shard_queue_[first_shard_queue_index]->shard_queue_index =
+      first_shard_queue_index;
+  shard_queue_[first_shard_queue_index + 1]->shard_queue_index =
+      first_shard_queue_index + 1;
+}
+
+void TimerList::NoteDeadlineChange(Shard* shard) {
+  while (shard->shard_queue_index > 0 &&
+         shard->min_deadline <
+             shard_queue_[shard->shard_queue_index - 1]->min_deadline) {
+    SwapAdjacentShardsInQueue(shard->shard_queue_index - 1);
+  }
+  while (shard->shard_queue_index < num_shards_ - 1 &&
+         shard->min_deadline >
+             shard_queue_[shard->shard_queue_index + 1]->min_deadline) {
+    SwapAdjacentShardsInQueue(shard->shard_queue_index);
+  }
+}
+
+void TimerList::TimerInit(Timer* timer, grpc_core::Timestamp deadline,
+                          experimental::EventEngine::Closure* closure) {
+  bool is_first_timer = false;
+  Shard* shard = &shards_[grpc_core::HashPointer(timer, num_shards_)];
+  timer->closure = closure;
+  timer->deadline = deadline.milliseconds_after_process_epoch();
+
+#ifndef NDEBUG
+  timer->hash_table_next = nullptr;
+#endif
+
+  {
+    grpc_core::MutexLock lock(&shard->mu);
+    timer->pending = true;
+    grpc_core::Timestamp now = host_->Now();
+    if (deadline <= now) {
+      deadline = now;
+    }
+
+    shard->stats.AddSample((deadline - now).millis() / 1000.0);
+
+    if (deadline < shard->queue_deadline_cap) {
+      is_first_timer = shard->heap.Add(timer);
+    } else {
+      timer->heap_index = kInvalidHeapIndex;
+      ListJoin(&shard->list, timer);
+    }
+  }
+
+  /* Deadline may have decreased, we need to adjust the main queue.  Note
+     that there is a potential racy unlocked region here.  There could be a
+     reordering of multiple TimerInit calls, at this point, but the < test
+     below should ensure that we err on the side of caution.  There could
+     also be a race with TimerCheck, which might beat us to the lock.  In
+     that case, it is possible that the timer that we added will have already
+     run by the time we hold the lock, but that too is a safe error.
+     Finally, it's possible that the TimerCheck that intervened failed to
+     trigger the new timer because the min_deadline hadn't yet been reduced.
+     In that case, the timer will simply have to wait for the next
+     TimerCheck. */
+  if (is_first_timer) {
+    grpc_core::MutexLock lock(&mu_);
+    if (deadline < shard->min_deadline) {
+      grpc_core::Timestamp old_min_deadline = shard_queue_[0]->min_deadline;
+      shard->min_deadline = deadline;
+      NoteDeadlineChange(shard);
+      if (shard->shard_queue_index == 0 && deadline < old_min_deadline) {
+        min_timer_.store(deadline.milliseconds_after_process_epoch(),
+                         std::memory_order_relaxed);
+        host_->Kick();
+      }
+    }
+  }
+}
+
+bool TimerList::TimerCancel(Timer* timer) {
+  Shard* shard = &shards_[grpc_core::HashPointer(timer, num_shards_)];
+  grpc_core::MutexLock lock(&shard->mu);
+
+  if (timer->pending) {
+    timer->pending = false;
+    if (timer->heap_index == kInvalidHeapIndex) {
+      ListRemove(timer);
+    } else {
+      shard->heap.Remove(timer);
+    }
+    return true;
+  }
+
+  return false;
+}
+
+/* Rebalances the timer shard by computing a new 'queue_deadline_cap' and moving
+   all relevant timers in shard->list (i.e timers with deadlines earlier than
+   'queue_deadline_cap') into into shard->heap.
+   Returns 'true' if shard->heap has at least ONE element */
+bool TimerList::Shard::RefillHeap(grpc_core::Timestamp now) {
+  /* Compute the new queue window width and bound by the limits: */
+  double computed_deadline_delta = stats.UpdateAverage() * kAddDeadlineScale;
+  double deadline_delta =
+      grpc_core::Clamp(computed_deadline_delta, kMinQueueWindowDuration,
+                       kMaxQueueWindowDuration);
+  Timer *timer, *next;
+
+  /* Compute the new cap and put all timers under it into the queue: */
+  queue_deadline_cap = std::max(now, queue_deadline_cap) +
+                       grpc_core::Duration::FromSecondsAsDouble(deadline_delta);
+
+  for (timer = list.next; timer != &list; timer = next) {
+    next = timer->next;
+    auto timer_deadline =
+        grpc_core::Timestamp::FromMillisecondsAfterProcessEpoch(
+            timer->deadline);
+
+    if (timer_deadline < queue_deadline_cap) {
+      ListRemove(timer);
+      heap.Add(timer);
+    }
+  }
+  return !heap.is_empty();
+}
+
+/* This pops the next non-cancelled timer with deadline <= now from the
+   queue, or returns NULL if there isn't one. */
+Timer* TimerList::Shard::PopOne(grpc_core::Timestamp now) {
+  Timer* timer;
+  for (;;) {
+    if (heap.is_empty()) {
+      if (now < queue_deadline_cap) return nullptr;
+      if (!RefillHeap(now)) return nullptr;
+    }
+    timer = heap.Top();
+    auto timer_deadline =
+        grpc_core::Timestamp::FromMillisecondsAfterProcessEpoch(
+            timer->deadline);
+    if (timer_deadline > now) return nullptr;
+    timer->pending = false;
+    heap.Pop();
+    return timer;
+  }
+}
+
+void TimerList::Shard::PopTimers(
+    grpc_core::Timestamp now, grpc_core::Timestamp* new_min_deadline,
+    std::vector<experimental::EventEngine::Closure*>* out) {
+  grpc_core::MutexLock lock(&mu);
+  while (Timer* timer = PopOne(now)) {
+    out->push_back(timer->closure);
+  }
+  *new_min_deadline = ComputeMinDeadline();
+}
+
+std::vector<experimental::EventEngine::Closure*> TimerList::FindExpiredTimers(
+    grpc_core::Timestamp now, grpc_core::Timestamp* next) {
+  grpc_core::Timestamp min_timer =
+      grpc_core::Timestamp::FromMillisecondsAfterProcessEpoch(
+          min_timer_.load(std::memory_order_relaxed));
+
+  std::vector<experimental::EventEngine::Closure*> done;
+  if (now < min_timer) {
+    if (next != nullptr) *next = std::min(*next, min_timer);
+    return done;
+  }
+
+  grpc_core::MutexLock lock(&mu_);
+
+  while (shard_queue_[0]->min_deadline < now ||
+         (now != grpc_core::Timestamp::InfFuture() &&
+          shard_queue_[0]->min_deadline == now)) {
+    grpc_core::Timestamp new_min_deadline;
+
+    /* For efficiency, we pop as many available timers as we can from the
+       shard.  This may violate perfect timer deadline ordering, but that
+       shouldn't be a big deal because we don't make ordering guarantees. */
+    shard_queue_[0]->PopTimers(now, &new_min_deadline, &done);
+
+    /* An TimerInit() on the shard could intervene here, adding a new
+       timer that is earlier than new_min_deadline.  However,
+       TimerInit() will block on the mutex before it can call
+       set_min_deadline, so this one will complete first and then the Addtimer
+       will reduce the min_deadline (perhaps unnecessarily). */
+    shard_queue_[0]->min_deadline = new_min_deadline;
+    NoteDeadlineChange(shard_queue_[0]);
+  }
+
+  if (next) {
+    *next = std::min(*next, shard_queue_[0]->min_deadline);
+  }
+
+  min_timer_.store(
+      shard_queue_[0]->min_deadline.milliseconds_after_process_epoch(),
+      std::memory_order_relaxed);
+
+  return done;
+}
+
+absl::optional<std::vector<experimental::EventEngine::Closure*>>
+TimerList::TimerCheck(grpc_core::Timestamp* next) {
+  // prelude
+  grpc_core::Timestamp now = host_->Now();
+
+  /* fetch from a thread-local first: this avoids contention on a globally
+     mutable cacheline in the common case */
+  grpc_core::Timestamp min_timer =
+      grpc_core::Timestamp::FromMillisecondsAfterProcessEpoch(
+          min_timer_.load(std::memory_order_relaxed));
+
+  if (now < min_timer) {
+    if (next != nullptr) {
+      *next = std::min(*next, min_timer);
+    }
+    return std::vector<experimental::EventEngine::Closure*>();
+  }
+
+  if (!checker_mu_.TryLock()) return absl::nullopt;
+  std::vector<experimental::EventEngine::Closure*> run =
+      FindExpiredTimers(now, next);
+  checker_mu_.Unlock();
+
+  return std::move(run);
+}
+
+}  // namespace iomgr_engine
+}  // namespace grpc_event_engine