grpc 1.49.0.pre1-x86_64-linux → 1.50.0.pre1-x86_64-linux

data/src/core/lib/event_engine/channel_args_endpoint_config.h

@@ -17,24 +17,30 @@
 #include <grpc/support/port_platform.h>
 
 #include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
 
 #include <grpc/event_engine/endpoint_config.h>
-#include <grpc/impl/codegen/grpc_types.h>
+
+#include "src/core/lib/channel/channel_args.h"
 
 namespace grpc_event_engine {
 namespace experimental {
 
-/// A readonly \a EndpointConfig based on grpc_channel_args. This class does not
-/// take ownership of the grpc_endpoint_args*, and instances of this class
-/// should not be used after the underlying args are destroyed.
 class ChannelArgsEndpointConfig : public EndpointConfig {
  public:
-  explicit ChannelArgsEndpointConfig(const grpc_channel_args* args)
+  ChannelArgsEndpointConfig() = default;
+  explicit ChannelArgsEndpointConfig(const grpc_core::ChannelArgs& args)
       : args_(args) {}
-  Setting Get(absl::string_view key) const override;
+  ChannelArgsEndpointConfig(const ChannelArgsEndpointConfig& config) = default;
+  ChannelArgsEndpointConfig& operator=(const ChannelArgsEndpointConfig& other) =
+      default;
+  absl::optional<int> GetInt(absl::string_view key) const override;
+  absl::optional<absl::string_view> GetString(
+      absl::string_view key) const override;
+  void* GetVoidPointer(absl::string_view key) const override;
 
  private:
-  const grpc_channel_args* args_;
+  grpc_core::ChannelArgs args_;
 };
 
 }  // namespace experimental

data/src/core/lib/event_engine/forkable.cc

@@ -22,15 +22,17 @@
 
 #include "absl/container/flat_hash_set.h"
 
+#include "src/core/lib/gprpp/no_destruct.h"
 #include "src/core/lib/gprpp/sync.h"
 
 namespace grpc_event_engine {
 namespace experimental {
 
 namespace {
-grpc_core::Mutex g_mu;
-bool g_registered{false};
-absl::flat_hash_set<Forkable*> g_forkables;
+grpc_core::NoDestruct<grpc_core::Mutex> g_mu;
+bool g_registered ABSL_GUARDED_BY(g_mu){false};
+grpc_core::NoDestruct<absl::flat_hash_set<Forkable*>> g_forkables
+    ABSL_GUARDED_BY(g_mu);
 }  // namespace
 
 Forkable::Forkable() { ManageForkable(this); }
@@ -38,39 +40,40 @@ Forkable::Forkable() { ManageForkable(this); }
 Forkable::~Forkable() { StopManagingForkable(this); }
 
 void RegisterForkHandlers() {
-  grpc_core::MutexLock lock(&g_mu);
-  GPR_ASSERT(!absl::exchange(g_registered, true));
-  pthread_atfork(PrepareFork, PostforkParent, PostforkChild);
+  grpc_core::MutexLock lock(g_mu.get());
+  if (!std::exchange(g_registered, true)) {
+    pthread_atfork(PrepareFork, PostforkParent, PostforkChild);
+  }
 };
 
 void PrepareFork() {
-  grpc_core::MutexLock lock(&g_mu);
-  for (auto* forkable : g_forkables) {
+  grpc_core::MutexLock lock(g_mu.get());
+  for (auto* forkable : *g_forkables) {
     forkable->PrepareFork();
   }
 }
 void PostforkParent() {
-  grpc_core::MutexLock lock(&g_mu);
-  for (auto* forkable : g_forkables) {
+  grpc_core::MutexLock lock(g_mu.get());
+  for (auto* forkable : *g_forkables) {
     forkable->PostforkParent();
   }
 }
 
 void PostforkChild() {
-  grpc_core::MutexLock lock(&g_mu);
-  for (auto* forkable : g_forkables) {
+  grpc_core::MutexLock lock(g_mu.get());
+  for (auto* forkable : *g_forkables) {
     forkable->PostforkChild();
   }
 }
 
 void ManageForkable(Forkable* forkable) {
-  grpc_core::MutexLock lock(&g_mu);
-  g_forkables.insert(forkable);
+  grpc_core::MutexLock lock(g_mu.get());
+  g_forkables->insert(forkable);
 }
 
 void StopManagingForkable(Forkable* forkable) {
-  grpc_core::MutexLock lock(&g_mu);
-  g_forkables.erase(forkable);
+  grpc_core::MutexLock lock(g_mu.get());
+  g_forkables->erase(forkable);
 }
 
 }  // namespace experimental

data/src/core/lib/event_engine/poller.h

@@ -16,8 +16,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "absl/container/inlined_vector.h"
-#include "absl/types/variant.h"
+#include "absl/functional/function_ref.h"
 
 #include <grpc/event_engine/event_engine.h>
 
@@ -30,20 +29,23 @@ namespace experimental {
 // Work(...).
 class Poller {
  public:
-  // This initial vector size may need to be tuned
-  using Events = absl::InlinedVector<EventEngine::Closure*, 5>;
-  struct DeadlineExceeded {};
-  struct Kicked {};
-  using WorkResult = absl::variant<Events, DeadlineExceeded, Kicked>;
+  enum class WorkResult { kOk, kDeadlineExceeded, kKicked };
 
   virtual ~Poller() = default;
-  // Poll once for events, returning a collection of Closures to be executed.
+  // Poll once for events and process received events. The callback function
+  // "schedule_poll_again" is expected to be run synchronously prior to
+  // processing received events. The callback's responsibility primarily is to
+  // schedule Poller::Work asynchronously again. This would ensure that the next
+  // polling cycle would run as quickly as possible to ensure continuous
+  // polling.
   //
   // Returns:
-  //  * absl::AbortedError if it was Kicked.
-  //  * absl::DeadlineExceeded if timeout occurred
-  //  * A collection of closures to execute, otherwise
-  virtual WorkResult Work(EventEngine::Duration timeout) = 0;
+  //  * Poller::WorkResult::kKicked if it was Kicked.
+  //  * Poller::WorkResult::kDeadlineExceeded if timeout occurred
+  //  * Poller::WorkResult::kOk, otherwise indicating that the callback function
+  //  was run synchonously before some events were processed.
+  virtual WorkResult Work(EventEngine::Duration timeout,
+                          absl::FunctionRef<void()> schedule_poll_again) = 0;
   // Trigger the threads executing Work(..) to break out as soon as possible.
   virtual void Kick() = 0;
 };

data/src/core/lib/event_engine/posix_engine/timer_manager.cc

@@ -32,11 +32,17 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gprpp/thd.h"
 
+static GPR_THREAD_LOCAL(bool) g_timer_thread;
+
 namespace grpc_event_engine {
 namespace posix_engine {
 
+grpc_core::DebugOnlyTraceFlag grpc_event_engine_timer_trace(false, "timer");
+
 namespace {
 class ThreadCollector {
  public:
@@ -88,7 +94,7 @@ void TimerManager::RunSomeTimers(
       // if there's no thread waiting with a timeout, kick an existing untimed
       // waiter so that the next deadline is not missed
       if (!has_timed_waiter_) {
-        cv_.Signal();
+        cv_wait_.Signal();
       }
     }
   }
@@ -151,8 +157,8 @@ bool TimerManager::WaitUntil(grpc_core::Timestamp next) {
       }
     }
 
-    cv_.WaitWithTimeout(&mu_,
-                        absl::Milliseconds((next - host_.Now()).millis()));
+    cv_wait_.WaitWithTimeout(&mu_,
+                             absl::Milliseconds((next - host_.Now()).millis()));
 
     // if this was the timed waiter, then we need to check timers, and flag
     // that there's now no timed waiter... we'll look for a replacement if
@@ -196,16 +202,29 @@ void TimerManager::MainLoop() {
 }
 
 void TimerManager::RunThread(void* arg) {
+  g_timer_thread = true;
   std::unique_ptr<RunThreadArgs> thread(static_cast<RunThreadArgs*>(arg));
-  thread->self->MainLoop();
-  {
-    grpc_core::MutexLock lock(&thread->self->mu_);
-    thread->self->thread_count_--;
-    thread->self->completed_threads_.push_back(std::move(thread->thread));
+  if (grpc_event_engine_timer_trace.enabled()) {
+    gpr_log(GPR_DEBUG, "TimerManager::%p starting thread::%p", thread->self,
+            &thread->thread);
   }
-  thread->self->cv_.Signal();
+  thread->self->Run(std::move(thread->thread));
+  if (grpc_event_engine_timer_trace.enabled()) {
+    gpr_log(GPR_DEBUG, "TimerManager::%p thread::%p finished", thread->self,
+            &thread->thread);
+  }
+}
+
+void TimerManager::Run(grpc_core::Thread thread) {
+  MainLoop();
+  grpc_core::MutexLock lock(&mu_);
+  completed_threads_.push_back(std::move(thread));
+  thread_count_--;
+  if (thread_count_ == 0) cv_threadcount_.Signal();
 }
 
+bool TimerManager::IsTimerManagerThread() { return g_timer_thread; }
+
 TimerManager::TimerManager() : host_(this) {
   timer_list_ = absl::make_unique<TimerList>(&host_);
   grpc_core::MutexLock lock(&mu_);
@@ -227,17 +246,23 @@ bool TimerManager::TimerCancel(Timer* timer) {
 }
 
 TimerManager::~TimerManager() {
-  {
-    grpc_core::MutexLock lock(&mu_);
-    shutdown_ = true;
-    cv_.SignalAll();
+  if (grpc_event_engine_timer_trace.enabled()) {
+    gpr_log(GPR_DEBUG, "TimerManager::%p shutting down", this);
   }
-  while (true) {
-    ThreadCollector collector;
-    grpc_core::MutexLock lock(&mu_);
-    collector.Collect(std::move(completed_threads_));
-    if (thread_count_ == 0) break;
-    cv_.Wait(&mu_);
+  ThreadCollector collector;
+  grpc_core::MutexLock lock(&mu_);
+  shutdown_ = true;
+  cv_wait_.SignalAll();
+  while (thread_count_ > 0) {
+    cv_threadcount_.Wait(&mu_);
+    if (grpc_event_engine_timer_trace.enabled()) {
+      gpr_log(GPR_DEBUG, "TimerManager::%p waiting for %zu threads to finish",
+              this, thread_count_);
+    }
+  }
+  collector.Collect(std::move(completed_threads_));
+  if (grpc_event_engine_timer_trace.enabled()) {
+    gpr_log(GPR_DEBUG, "TimerManager::%p shutdown complete", this);
   }
 }
 
@@ -249,23 +274,19 @@ void TimerManager::Kick() {
   timed_waiter_deadline_ = grpc_core::Timestamp::InfFuture();
   ++timed_waiter_generation_;
   kicked_ = true;
-  cv_.Signal();
+  cv_wait_.Signal();
 }
 
 void TimerManager::PrepareFork() {
-  {
-    grpc_core::MutexLock lock(&mu_);
-    forking_ = true;
-    prefork_thread_count_ = thread_count_;
-    cv_.SignalAll();
-  }
-  while (true) {
-    grpc_core::MutexLock lock(&mu_);
-    ThreadCollector collector;
-    collector.Collect(std::move(completed_threads_));
-    if (thread_count_ == 0) break;
-    cv_.Wait(&mu_);
+  ThreadCollector collector;
+  grpc_core::MutexLock lock(&mu_);
+  forking_ = true;
+  prefork_thread_count_ = thread_count_;
+  cv_wait_.SignalAll();
+  while (thread_count_ > 0) {
+    cv_threadcount_.Wait(&mu_);
   }
+  collector.Collect(std::move(completed_threads_));
 }
 
 void TimerManager::PostforkParent() {

data/src/core/lib/event_engine/posix_engine/timer_manager.h

@@ -60,6 +60,8 @@ class TimerManager final : public grpc_event_engine::experimental::Forkable {
   void PostforkParent() override;
   void PostforkChild() override;
 
+  static bool IsTimerManagerThread();
+
  private:
   struct RunThreadArgs {
     TimerManager* self;
@@ -80,13 +82,33 @@ class TimerManager final : public grpc_event_engine::experimental::Forkable {
 
   void StartThread() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
   static void RunThread(void* arg);
+  void Run(grpc_core::Thread thread);
   void MainLoop();
   void RunSomeTimers(std::vector<experimental::EventEngine::Closure*> timers);
   bool WaitUntil(grpc_core::Timestamp next);
   void Kick();
 
   grpc_core::Mutex mu_;
-  grpc_core::CondVar cv_;
+  // Condvar associated with decrementing the thread count.
+  // Threads will signal this when thread count reaches zero, and the forking
+  // code *or* the destructor will wait upon it.
+  grpc_core::CondVar cv_threadcount_;
+  // Condvar associated with threads waiting to wakeup and work.
+  // Threads wait on this until either a timeout is reached or another thread is
+  // needed to wait for a timeout.
+  // On shutdown we SignalAll against this to wake up all threads and have them
+  // finish.
+  // On kick we Signal against this to wake up at least one thread (but not
+  // all)! Similarly when we note that no thread is watching timers.
+  //
+  // This is a different condvar than cv_threadcount_!
+  // If this were the same:
+  // - thread exits would require a SignalAll to ensure that the specific thread
+  //   we want to wake is woken up.
+  // - kicks would need to signal all threads to avoid having the kick absorbed
+  //   by a shutdown thread and cause a deadlock, leading to thundering herd
+  //   problems in the common case.
+  grpc_core::CondVar cv_wait_;
   Host host_;
   // number of threads in the system
   size_t thread_count_ ABSL_GUARDED_BY(mu_) = 0;

data/src/core/lib/event_engine/thread_pool.cc

@@ -20,138 +20,175 @@
 
 #include "src/core/lib/event_engine/thread_pool.h"
 
+#include <memory>
 #include <utility>
 
+#include "absl/time/clock.h"
+#include "absl/time/time.h"
+
+#include <grpc/support/log.h>
+
+#include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gprpp/thd.h"
 
 namespace grpc_event_engine {
 namespace experimental {
 
-ThreadPool::Thread::Thread(ThreadPool* pool)
-    : pool_(pool),
-      thd_(
-          "posix_eventengine_pool",
-          [](void* th) { static_cast<ThreadPool::Thread*>(th)->ThreadFunc(); },
-          this, nullptr, grpc_core::Thread::Options().set_tracked(false)) {
-  thd_.Start();
+namespace {
+// TODO(drfloob): Remove this, and replace it with the WorkQueue* for the
+// current thread (with nullptr indicating not a threadpool thread).
+GPR_THREAD_LOCAL(bool) g_threadpool_thread;
+}  // namespace
+
+void ThreadPool::StartThread(StatePtr state, bool throttled) {
+  state->thread_count.Add();
+  struct ThreadArg {
+    StatePtr state;
+    bool throttled;
+  };
+  grpc_core::Thread(
+      "event_engine",
+      [](void* arg) {
+        std::unique_ptr<ThreadArg> a(static_cast<ThreadArg*>(arg));
+        g_threadpool_thread = true;
+        if (a->throttled) {
+          GPR_ASSERT(a->state->currently_starting_one_thread.exchange(
+              false, std::memory_order_relaxed));
+        }
+        ThreadFunc(a->state);
+      },
+      new ThreadArg{state, throttled}, nullptr,
+      grpc_core::Thread::Options().set_tracked(false).set_joinable(false))
+      .Start();
 }
-ThreadPool::Thread::~Thread() { thd_.Join(); }
-
-void ThreadPool::Thread::ThreadFunc() {
-  pool_->ThreadFunc();
-  // Now that we have killed ourselves, we should reduce the thread count
-  grpc_core::MutexLock lock(&pool_->mu_);
-  pool_->nthreads_--;
-  // Move ourselves to dead list
-  pool_->dead_threads_.push_back(this);
-
-  if (pool_->nthreads_ == 0) {
-    if (pool_->forking_) pool_->fork_cv_.Signal();
-    if (pool_->shutdown_) pool_->shutdown_cv_.Signal();
+
+void ThreadPool::ThreadFunc(StatePtr state) {
+  while (state->queue.Step()) {
   }
+  state->thread_count.Remove();
 }
 
-void ThreadPool::ThreadFunc() {
-  for (;;) {
-    // Wait until work is available or we are shutting down.
-    grpc_core::ReleasableMutexLock lock(&mu_);
-    if (!forking_ && !shutdown_ && callbacks_.empty()) {
-      // If there are too many threads waiting, then quit this thread
-      if (threads_waiting_ >= reserve_threads_) {
-        break;
-      }
-      threads_waiting_++;
-      cv_.Wait(&mu_);
-      threads_waiting_--;
-    }
-    // a fork could be initiated while the thread was waiting
-    if (forking_) return;
-    // Drain callbacks before considering shutdown to ensure all work
-    // gets completed.
-    if (!callbacks_.empty()) {
-      auto cb = std::move(callbacks_.front());
-      callbacks_.pop();
-      lock.Release();
-      cb();
-    } else if (shutdown_) {
+bool ThreadPool::Queue::Step() {
+  grpc_core::ReleasableMutexLock lock(&mu_);
+  // Wait until work is available or we are shutting down.
+  while (state_ == State::kRunning && callbacks_.empty()) {
+    // If there are too many threads waiting, then quit this thread.
+    // TODO(ctiller): wait some time in this case to be sure.
+    if (threads_waiting_ >= reserve_threads_) return false;
+    threads_waiting_++;
+    cv_.Wait(&mu_);
+    threads_waiting_--;
+  }
+  switch (state_) {
+    case State::kRunning:
       break;
-    }
+    case State::kShutdown:
+    case State::kForking:
+      if (!callbacks_.empty()) break;
+      return false;
   }
+  GPR_ASSERT(!callbacks_.empty());
+  auto callback = std::move(callbacks_.front());
+  callbacks_.pop();
+  lock.Release();
+  callback();
+  return true;
 }
 
 ThreadPool::ThreadPool(int reserve_threads)
-    : shutdown_(false),
-      reserve_threads_(reserve_threads),
-      nthreads_(0),
-      threads_waiting_(0),
-      forking_(false) {
-  grpc_core::MutexLock lock(&mu_);
-  StartNThreadsLocked(reserve_threads_);
-}
-
-void ThreadPool::StartNThreadsLocked(int n) {
-  for (int i = 0; i < n; i++) {
-    nthreads_++;
-    new Thread(this);
+    : reserve_threads_(reserve_threads) {
+  for (int i = 0; i < reserve_threads; i++) {
+    StartThread(state_, /*throttled=*/false);
   }
 }
 
-void ThreadPool::ReapThreads(std::vector<Thread*>* tlist) {
-  for (auto* t : *tlist) delete t;
-  tlist->clear();
+ThreadPool::~ThreadPool() {
+  state_->queue.SetShutdown();
+  // Wait until all threads are exited.
+  // Note that if this is a threadpool thread then we won't exit this thread
+  // until the callstack unwinds a little, so we need to wait for just one
+  // thread running instead of zero.
+  state_->thread_count.BlockUntilThreadCount(g_threadpool_thread ? 1 : 0,
+                                             "shutting down");
 }
 
-ThreadPool::~ThreadPool() {
-  grpc_core::MutexLock lock(&mu_);
-  shutdown_ = true;
-  cv_.SignalAll();
-  while (nthreads_ != 0) {
-    shutdown_cv_.Wait(&mu_);
+void ThreadPool::Add(absl::AnyInvocable<void()> callback) {
+  if (state_->queue.Add(std::move(callback))) {
+    if (!state_->currently_starting_one_thread.exchange(
+            true, std::memory_order_relaxed)) {
+      StartThread(state_, /*throttled=*/true);
+    }
   }
-  ReapThreads(&dead_threads_);
 }
 
-void ThreadPool::Add(absl::AnyInvocable<void()> callback) {
+bool ThreadPool::Queue::Add(absl::AnyInvocable<void()> callback) {
   grpc_core::MutexLock lock(&mu_);
   // Add works to the callbacks list
   callbacks_.push(std::move(callback));
-  // Store the callback for later if we are forking.
-  // TODO(hork): should we block instead?
-  if (forking_) return;
-  // Increase pool size or notify as needed
-  if (threads_waiting_ == 0) {
-    // Kick off a new thread
-    nthreads_++;
-    new Thread(this);
-  } else {
-    cv_.Signal();
-  }
-  // Also use this chance to harvest dead threads
-  if (!dead_threads_.empty()) {
-    ReapThreads(&dead_threads_);
+  cv_.Signal();
+  switch (state_) {
+    case State::kRunning:
+    case State::kShutdown:
+      return threads_waiting_ == 0;
+    case State::kForking:
+      return false;
   }
+  GPR_UNREACHABLE_CODE(return false);
 }
 
-void ThreadPool::PrepareFork() {
+void ThreadPool::Queue::SetState(State state) {
   grpc_core::MutexLock lock(&mu_);
-  forking_ = true;
-  cv_.SignalAll();
-  while (nthreads_ != 0) {
-    fork_cv_.Wait(&mu_);
+  if (state == State::kRunning) {
+    GPR_ASSERT(state_ != State::kRunning);
+  } else {
+    GPR_ASSERT(state_ == State::kRunning);
   }
-  ReapThreads(&dead_threads_);
+  state_ = state;
+  cv_.SignalAll();
 }
 
-void ThreadPool::PostforkParent() {
+void ThreadPool::ThreadCount::Add() {
   grpc_core::MutexLock lock(&mu_);
-  forking_ = false;
-  StartNThreadsLocked(reserve_threads_);
+  ++threads_;
 }
 
-void ThreadPool::PostforkChild() {
+void ThreadPool::ThreadCount::Remove() {
   grpc_core::MutexLock lock(&mu_);
-  forking_ = false;
-  StartNThreadsLocked(reserve_threads_);
+  --threads_;
+  cv_.Signal();
+}
+
+void ThreadPool::ThreadCount::BlockUntilThreadCount(int threads,
+                                                    const char* why) {
+  grpc_core::MutexLock lock(&mu_);
+  auto last_log = absl::Now();
+  while (threads_ > threads) {
+    // Wait for all threads to exit.
+    // At least once every three seconds (but no faster than once per second in
+    // the event of spurious wakeups) log a message indicating we're waiting to
+    // fork.
+    cv_.WaitWithTimeout(&mu_, absl::Seconds(3));
+    if (threads_ > threads && absl::Now() - last_log > absl::Seconds(1)) {
+      gpr_log(GPR_ERROR, "Waiting for thread pool to idle before %s", why);
+      last_log = absl::Now();
+    }
+  }
+}
+
+void ThreadPool::PrepareFork() {
+  state_->queue.SetForking();
+  state_->thread_count.BlockUntilThreadCount(0, "forking");
+}
+
+void ThreadPool::PostforkParent() { Postfork(); }
+
+void ThreadPool::PostforkChild() { Postfork(); }
+
+void ThreadPool::Postfork() {
+  state_->queue.Reset();
+  for (int i = 0; i < reserve_threads_; i++) {
+    StartThread(state_, /*throttled=*/false);
+  }
 }
 
 }  // namespace experimental