grpc 1.56.2 → 1.58.0

data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc

@@ -20,6 +20,8 @@
 
 #include "src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.h"
 
+#include <inttypes.h>
+
 #include <atomic>
 #include <memory>
 #include <utility>
@@ -39,23 +41,74 @@
 #include "src/core/lib/gprpp/thd.h"
 #include "src/core/lib/gprpp/time.h"
 
+// ## Thread Pool Fork-handling
+//
+// Thread-safety needs special attention with regard to fork() calls. The
+// Forkable system employs a pre- and post- fork callback system that does not
+// guarantee any ordering of execution. On fork() events, the thread pool does
+// the following:
+//
+// On pre-fork:
+// * the WorkStealingThreadPool triggers all threads to exit,
+// * all queued work is saved, and
+// * all threads will are down, including the Lifeguard thread.
+//
+// On post-fork:
+//  * all threads are restarted, including the Lifeguard thread, and
+//  * all previously-saved work is enqueued for execution.
+//
+// However, the queue may may get into trouble if one thread is attempting to
+// restart the thread pool while another thread is shutting it down. For that
+// reason, Quiesce and Start must be thread-safe, and Quiesce must wait for the
+// pool to be in a fully started state before it is allowed to continue.
+// Consider this potential ordering of events between Start and Quiesce:
+//
+//     ┌──────────┐
+//     │ Thread 1 │
+//     └────┬─────┘  ┌──────────┐
+//          │        │ Thread 2 │
+//          ▼        └────┬─────┘
+//        Start()         │
+//          │             ▼
+//          │        Quiesce()
+//          │        Wait for worker threads to exit
+//          │        Wait for the lifeguard thread to exit
+//          ▼
+//        Start the Lifeguard thread
+//        Start the worker threads
+//
+// Thread 2 will find no worker threads, and it will then want to wait on a
+// non-existent Lifeguard thread to finish. Trying a simple
+// `lifeguard_thread_.Join()` leads to memory access errors. This implementation
+// uses Notifications to coordinate startup and shutdown states.
+
 namespace grpc_event_engine {
 namespace experimental {
 
 namespace {
+// Maximum amount of time an extra thread is allowed to idle before being
+// reclaimed.
 constexpr grpc_core::Duration kIdleThreadLimit =
     grpc_core::Duration::Seconds(20);
+// Rate at which "Waiting for ..." logs should be printed while quiescing.
+constexpr size_t kBlockingQuiesceLogRateSeconds = 3;
+// Minumum time between thread creations.
 constexpr grpc_core::Duration kTimeBetweenThrottledThreadStarts =
     grpc_core::Duration::Seconds(1);
+// Minimum time a worker thread should sleep between checking for new work. Used
+// in backoff calculations to reduce vigilance when the pool is calm.
 constexpr grpc_core::Duration kWorkerThreadMinSleepBetweenChecks{
     grpc_core::Duration::Milliseconds(15)};
+// Maximum time a worker thread should sleep between checking for new work.
 constexpr grpc_core::Duration kWorkerThreadMaxSleepBetweenChecks{
     grpc_core::Duration::Seconds(3)};
+// Minimum time the lifeguard thread should sleep between checks. Used in
+// backoff calculations to reduce vigilance when the pool is calm.
 constexpr grpc_core::Duration kLifeguardMinSleepBetweenChecks{
     grpc_core::Duration::Milliseconds(15)};
+// Maximum time the lifeguard thread should sleep between checking for new work.
 constexpr grpc_core::Duration kLifeguardMaxSleepBetweenChecks{
     grpc_core::Duration::Seconds(1)};
-constexpr absl::Duration kSleepBetweenQuiesceCheck{absl::Milliseconds(10)};
 }  // namespace
 
 thread_local WorkQueue* g_local_queue = nullptr;
@@ -113,13 +166,13 @@ void WorkStealingThreadPool::PostforkChild() { pool_->Postfork(); }
 
 WorkStealingThreadPool::WorkStealingThreadPoolImpl::WorkStealingThreadPoolImpl(
     size_t reserve_threads)
-    : reserve_threads_(reserve_threads), lifeguard_() {}
+    : reserve_threads_(reserve_threads), lifeguard_(this) {}
 
 void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Start() {
-  lifeguard_.Start(shared_from_this());
   for (size_t i = 0; i < reserve_threads_; i++) {
     StartThread();
   }
+  lifeguard_.Start();
 }
 
 void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Run(
@@ -127,9 +180,11 @@ void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Run(
   GPR_DEBUG_ASSERT(quiesced_.load(std::memory_order_relaxed) == false);
   if (g_local_queue != nullptr) {
     g_local_queue->Add(closure);
-    return;
+  } else {
+    queue_.Add(closure);
   }
-  queue_.Add(closure);
+  // Signal a worker in any case, even if work was added to a local queue. This
+  // improves performance on 32-core streaming benchmarks with small payloads.
   work_signal_.Signal();
 }
 
@@ -156,12 +211,12 @@ void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Quiesce() {
   // until all other threads have exited, so we need to wait for just one thread
   // running instead of zero.
   bool is_threadpool_thread = g_local_queue != nullptr;
-  thread_count()->BlockUntilThreadCount(CounterType::kLivingThreadCount,
-                                        is_threadpool_thread ? 1 : 0,
-                                        "shutting down", work_signal());
+  work_signal()->SignalAll();
+  living_thread_count_.BlockUntilThreadCount(is_threadpool_thread ? 1 : 0,
+                                             "shutting down");
   GPR_ASSERT(queue_.Empty());
   quiesced_.store(true, std::memory_order_relaxed);
-  lifeguard_.BlockUntilShutdown();
+  lifeguard_.BlockUntilShutdownAndReset();
 }
 
 bool WorkStealingThreadPool::WorkStealingThreadPoolImpl::SetThrottled(
@@ -196,9 +251,9 @@ bool WorkStealingThreadPool::WorkStealingThreadPoolImpl::IsQuiesced() {
 
 void WorkStealingThreadPool::WorkStealingThreadPoolImpl::PrepareFork() {
   SetForking(true);
-  thread_count()->BlockUntilThreadCount(CounterType::kLivingThreadCount, 0,
-                                        "forking", &work_signal_);
-  lifeguard_.BlockUntilShutdown();
+  work_signal_.SignalAll();
+  living_thread_count_.BlockUntilThreadCount(0, "forking");
+  lifeguard_.BlockUntilShutdownAndReset();
 }
 
 void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Postfork() {
@@ -206,21 +261,22 @@ void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Postfork() {
   Start();
 }
 
-// -------- WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard
-// --------
+// -------- WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard -----
 
-WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::Lifeguard()
-    : backoff_(grpc_core::BackOff::Options()
+WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::Lifeguard(
+    WorkStealingThreadPoolImpl* pool)
+    : pool_(pool),
+      backoff_(grpc_core::BackOff::Options()
                    .set_initial_backoff(kLifeguardMinSleepBetweenChecks)
                    .set_max_backoff(kLifeguardMaxSleepBetweenChecks)
-                   .set_multiplier(1.3)) {}
-
-void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::Start(
-    std::shared_ptr<WorkStealingThreadPoolImpl> pool) {
-  // thread_running_ is set early to avoid a quiesce race while the lifeguard is
-  // still starting up.
-  thread_running_.store(true);
-  pool_ = std::move(pool);
+                   .set_multiplier(1.3)),
+      lifeguard_should_shut_down_(std::make_unique<grpc_core::Notification>()),
+      lifeguard_is_shut_down_(std::make_unique<grpc_core::Notification>()) {}
+
+void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::Start() {
+  // lifeguard_running_ is set early to avoid a quiesce race while the
+  // lifeguard is still starting up.
+  lifeguard_running_.store(true);
   grpc_core::Thread(
       "lifeguard",
       [](void* arg) {
@@ -235,21 +291,39 @@ void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::Start(
 void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::
     LifeguardMain() {
   while (true) {
-    absl::SleepFor(absl::Milliseconds(
-        (backoff_.NextAttemptTime() - grpc_core::Timestamp::Now()).millis()));
     if (pool_->IsForking()) break;
-    if (pool_->IsShutdown() && pool_->IsQuiesced()) break;
+    // If the pool is shut down, loop quickly until quiesced. Otherwise,
+    // reduce the check rate if the pool is idle.
+    if (pool_->IsShutdown()) {
+      if (pool_->IsQuiesced()) break;
+    } else {
+      lifeguard_should_shut_down_->WaitForNotificationWithTimeout(
+          absl::Milliseconds(
+              (backoff_.NextAttemptTime() - grpc_core::Timestamp::Now())
+                  .millis()));
+    }
     MaybeStartNewThread();
   }
-  pool_.reset();
-  thread_running_.store(false);
+  lifeguard_running_.store(false, std::memory_order_relaxed);
+  lifeguard_is_shut_down_->Notify();
 }
 
 void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::
-    BlockUntilShutdown() {
-  while (thread_running_.load()) {
-    absl::SleepFor(kSleepBetweenQuiesceCheck);
+    BlockUntilShutdownAndReset() {
+  lifeguard_should_shut_down_->Notify();
+  while (lifeguard_running_.load(std::memory_order_relaxed)) {
+    GRPC_LOG_EVERY_N_SEC_DELAYED(kBlockingQuiesceLogRateSeconds, GPR_DEBUG,
+                                 "%s",
+                                 "Waiting for lifeguard thread to shut down");
+    lifeguard_is_shut_down_->WaitForNotification();
   }
+  // Do an additional wait in case this method races with LifeguardMain's
+  // shutdown. This should return immediately if the lifeguard is already shut
+  // down.
+  lifeguard_is_shut_down_->WaitForNotification();
+  backoff_.Reset();
+  lifeguard_should_shut_down_ = std::make_unique<grpc_core::Notification>();
+  lifeguard_is_shut_down_ = std::make_unique<grpc_core::Notification>();
 }
 
 void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::
@@ -257,12 +331,9 @@ void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::
   // No new threads are started when forking.
   // No new work is done when forking needs to begin.
   if (pool_->forking_.load()) return;
-  int busy_thread_count =
-      pool_->thread_count_.GetCount(CounterType::kBusyCount);
-  int living_thread_count =
-      pool_->thread_count_.GetCount(CounterType::kLivingThreadCount);
+  const auto living_thread_count = pool_->living_thread_count()->count();
   // Wake an idle worker thread if there's global work to be had.
-  if (busy_thread_count < living_thread_count) {
+  if (pool_->busy_thread_count()->count() < living_thread_count) {
     if (!pool_->queue_.Empty()) {
       pool_->work_signal()->Signal();
       backoff_.Reset();
@@ -285,7 +356,8 @@ void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::
   // queue, nor any work to steal. Add more sophisticated logic about when to
   // start a thread.
   GRPC_EVENT_ENGINE_TRACE(
-      "Starting new ThreadPool thread due to backlog (total threads: %d)",
+      "Starting new ThreadPool thread due to backlog (total threads: %" PRIuPTR
+      ")",
       living_thread_count + 1);
   pool_->StartThread();
   // Tell the lifeguard to monitor the pool more closely.
@@ -297,12 +369,13 @@ void WorkStealingThreadPool::WorkStealingThreadPoolImpl::Lifeguard::
 WorkStealingThreadPool::ThreadState::ThreadState(
     std::shared_ptr<WorkStealingThreadPoolImpl> pool)
     : pool_(std::move(pool)),
-      auto_thread_count_(pool_->thread_count(),
-                         CounterType::kLivingThreadCount),
+      auto_thread_counter_(
+          pool_->living_thread_count()->MakeAutoThreadCounter()),
       backoff_(grpc_core::BackOff::Options()
                    .set_initial_backoff(kWorkerThreadMinSleepBetweenChecks)
                    .set_max_backoff(kWorkerThreadMaxSleepBetweenChecks)
-                   .set_multiplier(1.3)) {}
+                   .set_multiplier(1.3)),
+      busy_count_idx_(pool_->busy_thread_count()->NextIndex()) {}
 
 void WorkStealingThreadPool::ThreadState::ThreadBody() {
   g_local_queue = new BasicWorkQueue();
@@ -340,8 +413,8 @@ bool WorkStealingThreadPool::ThreadState::Step() {
   auto* closure = g_local_queue->PopMostRecent();
   // If local work is available, run it.
   if (closure != nullptr) {
-    ThreadCount::AutoThreadCount auto_busy{pool_->thread_count(),
-                                           CounterType::kBusyCount};
+    auto busy =
+        pool_->busy_thread_count()->MakeAutoThreadCounter(busy_count_idx_);
     closure->Run();
     return true;
   }
@@ -374,11 +447,11 @@ bool WorkStealingThreadPool::ThreadState::Step() {
     if (pool_->IsShutdown()) break;
     bool timed_out = pool_->work_signal()->WaitWithTimeout(
         backoff_.NextAttemptTime() - grpc_core::Timestamp::Now());
+    if (pool_->IsForking() || pool_->IsShutdown()) break;
     // Quit a thread if the pool has more than it requires, and this thread
     // has been idle long enough.
     if (timed_out &&
-        pool_->thread_count()->GetCount(CounterType::kLivingThreadCount) >
-            pool_->reserve_threads() &&
+        pool_->living_thread_count()->count() > pool_->reserve_threads() &&
         grpc_core::Timestamp::Now() - start_time > kIdleThreadLimit) {
       return false;
     }
@@ -389,8 +462,8 @@ bool WorkStealingThreadPool::ThreadState::Step() {
     return false;
   }
   if (closure != nullptr) {
-    ThreadCount::AutoThreadCount auto_busy{pool_->thread_count(),
-                                           CounterType::kBusyCount};
+    auto busy =
+        pool_->busy_thread_count()->MakeAutoThreadCounter(busy_count_idx_);
     closure->Run();
   }
   backoff_.Reset();
@@ -399,8 +472,8 @@ bool WorkStealingThreadPool::ThreadState::Step() {
 
 void WorkStealingThreadPool::ThreadState::FinishDraining() {
   // The thread is definitionally busy while draining
-  ThreadCount::AutoThreadCount auto_busy{pool_->thread_count(),
-                                         CounterType::kBusyCount};
+  auto busy =
+      pool_->busy_thread_count()->MakeAutoThreadCounter(busy_count_idx_);
   // If a fork occurs at any point during shutdown, quit draining. The post-fork
   // threads will finish draining the global queue.
   while (!pool_->IsForking()) {
@@ -422,51 +495,6 @@ void WorkStealingThreadPool::ThreadState::FinishDraining() {
   }
 }
 
-// -------- WorkStealingThreadPool::ThreadCount --------
-
-void WorkStealingThreadPool::ThreadCount::Add(CounterType counter_type) {
-  thread_counts_[counter_type].fetch_add(1, std::memory_order_relaxed);
-}
-
-void WorkStealingThreadPool::ThreadCount::Remove(CounterType counter_type) {
-  thread_counts_[counter_type].fetch_sub(1, std::memory_order_relaxed);
-}
-
-void WorkStealingThreadPool::ThreadCount::BlockUntilThreadCount(
-    CounterType counter_type, int desired_threads, const char* why,
-    WorkSignal* work_signal) {
-  auto& counter = thread_counts_[counter_type];
-  int curr_threads = counter.load(std::memory_order_relaxed);
-  // Wait for all threads to exit.
-  auto last_log_time = grpc_core::Timestamp::Now();
-  while (curr_threads > desired_threads) {
-    absl::SleepFor(kSleepBetweenQuiesceCheck);
-    work_signal->SignalAll();
-    if (grpc_core::Timestamp::Now() - last_log_time >
-        grpc_core::Duration::Seconds(3)) {
-      gpr_log(GPR_DEBUG,
-              "Waiting for thread pool to idle before %s. (%d to %d)", why,
-              curr_threads, desired_threads);
-      last_log_time = grpc_core::Timestamp::Now();
-    }
-    curr_threads = counter.load(std::memory_order_relaxed);
-  }
-}
-
-size_t WorkStealingThreadPool::ThreadCount::GetCount(CounterType counter_type) {
-  return thread_counts_[counter_type].load(std::memory_order_relaxed);
-}
-
-WorkStealingThreadPool::ThreadCount::AutoThreadCount::AutoThreadCount(
-    ThreadCount* counter, CounterType counter_type)
-    : counter_(counter), counter_type_(counter_type) {
-  counter_->Add(counter_type_);
-}
-
-WorkStealingThreadPool::ThreadCount::AutoThreadCount::~AutoThreadCount() {
-  counter_->Remove(counter_type_);
-}
-
 // -------- WorkStealingThreadPool::WorkSignal --------
 
 void WorkStealingThreadPool::WorkSignal::Signal() {

data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.h

@@ -33,9 +33,11 @@
 #include <grpc/event_engine/event_engine.h>
 
 #include "src/core/lib/backoff/backoff.h"
+#include "src/core/lib/event_engine/thread_pool/thread_count.h"
 #include "src/core/lib/event_engine/thread_pool/thread_pool.h"
 #include "src/core/lib/event_engine/work_queue/basic_work_queue.h"
 #include "src/core/lib/event_engine/work_queue/work_queue.h"
+#include "src/core/lib/gprpp/notification.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/gprpp/time.h"
 
@@ -75,41 +77,6 @@ class WorkStealingThreadPool final : public ThreadPool {
     grpc_core::CondVar cv_ ABSL_GUARDED_BY(mu_);
   };
 
-  // Types of thread counts.
-  // Note this is intentionally not an enum class, the keys are used as indexes
-  // into the ThreadCount's private array.
-  enum CounterType {
-    kLivingThreadCount = 0,
-    kBusyCount,
-  };
-
-  class ThreadCount {
-   public:
-    // Adds 1 to the thread count for that counter type.
-    void Add(CounterType counter_type);
-    // Subtracts 1 from the thread count for that counter type.
-    void Remove(CounterType counter_type);
-    // Blocks until the thread count for that type reaches `desired_threads`.
-    void BlockUntilThreadCount(CounterType counter_type, int desired_threads,
-                               const char* why, WorkSignal* work_signal);
-    // Returns the current thread count for the tracked type.
-    size_t GetCount(CounterType counter_type);
-
-    // Adds and removes thread counts on construction and destruction
-    class AutoThreadCount {
-     public:
-      AutoThreadCount(ThreadCount* counter, CounterType counter_type);
-      ~AutoThreadCount();
-
-     private:
-      ThreadCount* counter_;
-      CounterType counter_type_;
-    };
-
-   private:
-    std::atomic<size_t> thread_counts_[2]{{0}, {0}};
-  };
-
   // A pool of WorkQueues that participate in work stealing.
   //
   // Every worker thread registers and unregisters its thread-local thread pool
@@ -152,8 +119,8 @@ class WorkStealingThreadPool final : public ThreadPool {
     // This method is safe to call from within a ThreadPool thread.
     void Quiesce();
     // Sets a throttled state.
-    // After the initial pool has been created, if the pool is backlogged when a
-    // new thread has started, it is rate limited.
+    // After the initial pool has been created, if the pool is backlogged when
+    // a new thread has started, it is rate limited.
     // Returns the previous throttling state.
     bool SetThrottled(bool throttle);
     // Set the shutdown flag.
@@ -170,7 +137,8 @@ class WorkStealingThreadPool final : public ThreadPool {
     bool IsForking();
     bool IsQuiesced();
     size_t reserve_threads() { return reserve_threads_; }
-    ThreadCount* thread_count() { return &thread_count_; }
+    BusyThreadCount* busy_thread_count() { return &busy_thread_count_; }
+    LivingThreadCount* living_thread_count() { return &living_thread_count_; }
     TheftRegistry* theft_registry() { return &theft_registry_; }
     WorkQueue* queue() { return &queue_; }
     WorkSignal* work_signal() { return &work_signal_; }
@@ -183,24 +151,30 @@ class WorkStealingThreadPool final : public ThreadPool {
     //    and there are threads that can accept work.
     class Lifeguard {
      public:
-      Lifeguard();
+      explicit Lifeguard(WorkStealingThreadPoolImpl* pool);
       // Start the lifeguard thread.
-      void Start(std::shared_ptr<WorkStealingThreadPoolImpl> pool);
+      void Start();
       // Block until the lifeguard thread is shut down.
-      void BlockUntilShutdown();
+      // Afterwards, reset the lifeguard state so it can start again cleanly.
+      void BlockUntilShutdownAndReset();
 
      private:
       // The main body of the lifeguard thread.
       void LifeguardMain();
       // Starts a new thread if the pool is backlogged
       void MaybeStartNewThread();
-      std::shared_ptr<WorkStealingThreadPoolImpl> pool_;
+
+      WorkStealingThreadPoolImpl* pool_;
       grpc_core::BackOff backoff_;
-      std::atomic<bool> thread_running_{false};
+      // Used for signaling that the lifeguard thread has stopped running.
+      std::unique_ptr<grpc_core::Notification> lifeguard_should_shut_down_;
+      std::unique_ptr<grpc_core::Notification> lifeguard_is_shut_down_;
+      std::atomic<bool> lifeguard_running_{false};
     };
 
     const size_t reserve_threads_;
-    ThreadCount thread_count_;
+    BusyThreadCount busy_thread_count_;
+    LivingThreadCount living_thread_count_;
     TheftRegistry theft_registry_;
     BasicWorkQueue queue_;
     // Track shutdown and fork bits separately.
@@ -233,11 +207,11 @@ class WorkStealingThreadPool final : public ThreadPool {
     // is decremented at time of destruction. This is necessary when this thread
     // state holds the last shared_ptr keeping the pool alive.
     std::shared_ptr<WorkStealingThreadPoolImpl> pool_;
-    // auto_thread_count_ must be the second member declared, so that the thread
-    // count is decremented after all other state is cleaned up (preventing
-    // leaks).
-    ThreadCount::AutoThreadCount auto_thread_count_;
+    // auto_thread_counter_ must be declared after pool_, so that the thread
+    // count is decremented after all other pool state is cleaned up.
+    LivingThreadCount::AutoThreadCounter auto_thread_counter_;
     grpc_core::BackOff backoff_;
+    size_t busy_count_idx_;
   };
 
   const std::shared_ptr<WorkStealingThreadPoolImpl> pool_;

data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc

@@ -82,9 +82,10 @@ bool ThreadyEventEngine::IsWorkerThread() {
   grpc_core::Crash("we should remove this");
 }
 
-std::unique_ptr<EventEngine::DNSResolver> ThreadyEventEngine::GetDNSResolver(
+absl::StatusOr<std::unique_ptr<EventEngine::DNSResolver>>
+ThreadyEventEngine::GetDNSResolver(
     const DNSResolver::ResolverOptions& options) {
-  return std::make_unique<ThreadyDNSResolver>(impl_->GetDNSResolver(options));
+  return std::make_unique<ThreadyDNSResolver>(*impl_->GetDNSResolver(options));
 }
 
 void ThreadyEventEngine::Run(Closure* closure) {
@@ -111,10 +112,9 @@ bool ThreadyEventEngine::Cancel(TaskHandle handle) {
   return impl_->Cancel(handle);
 }
 
-EventEngine::DNSResolver::LookupTaskHandle
-ThreadyEventEngine::ThreadyDNSResolver::LookupHostname(
+void ThreadyEventEngine::ThreadyDNSResolver::LookupHostname(
     LookupHostnameCallback on_resolve, absl::string_view name,
-    absl::string_view default_port, Duration timeout) {
+    absl::string_view default_port) {
   return impl_->LookupHostname(
       [this, on_resolve = std::move(on_resolve)](
           absl::StatusOr<std::vector<ResolvedAddress>> addresses) mutable {
@@ -123,13 +123,11 @@ ThreadyEventEngine::ThreadyDNSResolver::LookupHostname(
           on_resolve(std::move(addresses));
         });
       },
-      name, default_port, timeout);
+      name, default_port);
 }
 
-EventEngine::DNSResolver::LookupTaskHandle
-ThreadyEventEngine::ThreadyDNSResolver::LookupSRV(LookupSRVCallback on_resolve,
-                                                  absl::string_view name,
-                                                  Duration timeout) {
+void ThreadyEventEngine::ThreadyDNSResolver::LookupSRV(
+    LookupSRVCallback on_resolve, absl::string_view name) {
   return impl_->LookupSRV(
       [this, on_resolve = std::move(on_resolve)](
           absl::StatusOr<std::vector<SRVRecord>> records) mutable {
@@ -139,13 +137,11 @@ ThreadyEventEngine::ThreadyDNSResolver::LookupSRV(LookupSRVCallback on_resolve,
               on_resolve(std::move(records));
             });
       },
-      name, timeout);
+      name);
 }
 
-EventEngine::DNSResolver::LookupTaskHandle
-ThreadyEventEngine::ThreadyDNSResolver::LookupTXT(LookupTXTCallback on_resolve,
-                                                  absl::string_view name,
-                                                  Duration timeout) {
+void ThreadyEventEngine::ThreadyDNSResolver::LookupTXT(
+    LookupTXTCallback on_resolve, absl::string_view name) {
   return impl_->LookupTXT(
       [this, on_resolve = std::move(on_resolve)](
           absl::StatusOr<std::vector<std::string>> record) mutable {
@@ -154,12 +150,7 @@ ThreadyEventEngine::ThreadyDNSResolver::LookupTXT(LookupTXTCallback on_resolve,
           on_resolve(std::move(record));
         });
       },
-      name, timeout);
-}
-
-bool ThreadyEventEngine::ThreadyDNSResolver::CancelLookup(
-    LookupTaskHandle handle) {
-  return impl_->CancelLookup(handle);
+      name);
 }
 
 }  // namespace experimental

data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.h

@@ -63,7 +63,7 @@ class ThreadyEventEngine final : public EventEngine {
 
   bool IsWorkerThread() override;
 
-  std::unique_ptr<DNSResolver> GetDNSResolver(
+  absl::StatusOr<std::unique_ptr<DNSResolver>> GetDNSResolver(
       const DNSResolver::ResolverOptions& options) override;
 
   void Run(Closure* closure) override;
@@ -80,17 +80,13 @@ class ThreadyEventEngine final : public EventEngine {
    public:
     explicit ThreadyDNSResolver(std::unique_ptr<DNSResolver> impl)
         : impl_(std::move(impl)) {}
-    LookupTaskHandle LookupHostname(LookupHostnameCallback on_resolve,
-                                    absl::string_view name,
-                                    absl::string_view default_port,
-                                    Duration timeout) override;
-    LookupTaskHandle LookupSRV(LookupSRVCallback on_resolve,
-                               absl::string_view name,
-                               Duration timeout) override;
-    LookupTaskHandle LookupTXT(LookupTXTCallback on_resolve,
-                               absl::string_view name,
-                               Duration timeout) override;
-    bool CancelLookup(LookupTaskHandle handle) override;
+    void LookupHostname(LookupHostnameCallback on_resolve,
+                        absl::string_view name,
+                        absl::string_view default_port) override;
+    void LookupSRV(LookupSRVCallback on_resolve,
+                   absl::string_view name) override;
+    void LookupTXT(LookupTXTCallback on_resolve,
+                   absl::string_view name) override;
 
    private:
     std::unique_ptr<DNSResolver> impl_;