grpc 1.48.0-x86_64-linux → 1.49.0.pre1-x86_64-linux

data/src/core/lib/event_engine/{iomgr_engine → posix_engine}/timer_manager.cc

@@ -18,7 +18,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/lib/event_engine/iomgr_engine/timer_manager.h"
+#include "src/core/lib/event_engine/posix_engine/timer_manager.h"
 
 #include <algorithm>
 #include <memory>
@@ -35,7 +35,7 @@
 #include "src/core/lib/gprpp/thd.h"
 
 namespace grpc_event_engine {
-namespace iomgr_engine {
+namespace posix_engine {
 
 namespace {
 class ThreadCollector {
@@ -62,8 +62,9 @@ void TimerManager::StartThread() {
   ++thread_count_;
   auto* thread = new RunThreadArgs();
   thread->self = this;
-  thread->thread =
-      grpc_core::Thread("timer_manager", &TimerManager::RunThread, thread);
+  thread->thread = grpc_core::Thread(
+      "timer_manager", &TimerManager::RunThread, thread, nullptr,
+      grpc_core::Thread::Options().set_tracked(false));
   thread->thread.Start();
 }
 
@@ -73,12 +74,14 @@ void TimerManager::RunSomeTimers(
   ThreadCollector collector;
   {
     grpc_core::MutexLock lock(&mu_);
+    if (shutdown_ || forking_) return;
     // remove a waiter from the pool, and start another thread if necessary
     --waiter_count_;
     if (waiter_count_ == 0) {
       // The number of timer threads is always increasing until all the threads
-      // are stopped. In rare cases, if a large number of timers fire
-      // simultaneously, we may end up using a large number of threads.
+      // are stopped, with the exception that all threads are shut down on fork
+      // events. In rare cases, if a large number of timers fire simultaneously,
+      // we may end up using a large number of threads.
       // TODO(ctiller): We could avoid this by exiting threads in WaitUntil().
       StartThread();
     } else {
@@ -106,9 +109,8 @@ void TimerManager::RunSomeTimers(
 bool TimerManager::WaitUntil(grpc_core::Timestamp next) {
   grpc_core::MutexLock lock(&mu_);
 
-  if (shutdown_) {
-    return false;
-  }
+  if (shutdown_) return false;
+  if (forking_) return false;
 
   // TODO(ctiller): if there are too many waiting threads, this would be a good
   // place to exit the current thread.
@@ -250,5 +252,39 @@ void TimerManager::Kick() {
   cv_.Signal();
 }
 
-}  // namespace iomgr_engine
+void TimerManager::PrepareFork() {
+  {
+    grpc_core::MutexLock lock(&mu_);
+    forking_ = true;
+    prefork_thread_count_ = thread_count_;
+    cv_.SignalAll();
+  }
+  while (true) {
+    grpc_core::MutexLock lock(&mu_);
+    ThreadCollector collector;
+    collector.Collect(std::move(completed_threads_));
+    if (thread_count_ == 0) break;
+    cv_.Wait(&mu_);
+  }
+}
+
+void TimerManager::PostforkParent() {
+  grpc_core::MutexLock lock(&mu_);
+  for (int i = 0; i < prefork_thread_count_; i++) {
+    StartThread();
+  }
+  prefork_thread_count_ = 0;
+  forking_ = false;
+}
+
+void TimerManager::PostforkChild() {
+  grpc_core::MutexLock lock(&mu_);
+  for (int i = 0; i < prefork_thread_count_; i++) {
+    StartThread();
+  }
+  prefork_thread_count_ = 0;
+  forking_ = false;
+}
+
+}  // namespace posix_engine
 }  // namespace grpc_event_engine

data/src/core/lib/event_engine/{iomgr_engine → posix_engine}/timer_manager.h

@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIMER_MANAGER_H
-#define GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIMER_MANAGER_H
+#ifndef GRPC_CORE_LIB_EVENT_ENGINE_POSIX_ENGINE_TIMER_MANAGER_H
+#define GRPC_CORE_LIB_EVENT_ENGINE_POSIX_ENGINE_TIMER_MANAGER_H
 
 #include <grpc/support/port_platform.h>
 
@@ -31,22 +31,23 @@
 
 #include <grpc/event_engine/event_engine.h>
 
-#include "src/core/lib/event_engine/iomgr_engine/timer.h"
+#include "src/core/lib/event_engine/forkable.h"
+#include "src/core/lib/event_engine/posix_engine/timer.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/gprpp/thd.h"
 #include "src/core/lib/gprpp/time.h"
 
 namespace grpc_event_engine {
-namespace iomgr_engine {
+namespace posix_engine {
 
 // Timer Manager tries to keep only one thread waiting for the next timeout at
 // all times, and thus effectively preventing the thundering herd problem.
 // TODO(ctiller): consider unifying this thread pool and the one in
 // thread_pool.{h,cc}.
-class TimerManager final {
+class TimerManager final : public grpc_event_engine::experimental::Forkable {
  public:
   TimerManager();
-  ~TimerManager();
+  ~TimerManager() override;
 
   grpc_core::Timestamp Now() { return host_.Now(); }
 
@@ -54,6 +55,11 @@ class TimerManager final {
                  experimental::EventEngine::Closure* closure);
   bool TimerCancel(Timer* timer);
 
+  // Forkable
+  void PrepareFork() override;
+  void PostforkParent() override;
+  void PostforkChild() override;
+
  private:
   struct RunThreadArgs {
     TimerManager* self;
@@ -92,6 +98,8 @@ class TimerManager final {
   bool has_timed_waiter_ ABSL_GUARDED_BY(mu_) = false;
   // are we shutting down?
   bool shutdown_ ABSL_GUARDED_BY(mu_) = false;
+  // are we forking?
+  bool forking_ ABSL_GUARDED_BY(mu_) = false;
   // are we shutting down?
   bool kicked_ ABSL_GUARDED_BY(mu_) = false;
   // the deadline of the current timed waiter thread (only relevant if
@@ -103,9 +111,10 @@ class TimerManager final {
   uint64_t wakeups_ ABSL_GUARDED_BY(mu_) = 0;
   // actual timer implementation
   std::unique_ptr<TimerList> timer_list_;
+  int prefork_thread_count_ = 0;
 };
 
-}  // namespace iomgr_engine
+}  // namespace posix_engine
 }  // namespace grpc_event_engine
 
-#endif /* GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_TIMER_MANAGER_H */
+#endif /* GRPC_CORE_LIB_EVENT_ENGINE_POSIX_ENGINE_TIMER_MANAGER_H */

data/src/core/lib/event_engine/promise.h

@@ -15,6 +15,8 @@
 #define GRPC_CORE_LIB_EVENT_ENGINE_PROMISE_H
 #include <grpc/support/port_platform.h>
 
+#include "absl/time/time.h"
+
 #include <grpc/support/log.h>
 
 #include "src/core/lib/gprpp/sync.h"
@@ -30,12 +32,19 @@ namespace experimental {
 template <typename T>
 class Promise {
  public:
+  Promise() = default;
+  // Initialize a default value that will be returned if WaitWithTimeout times
+  // out
+  explicit Promise(T&& val) : val_(val) {}
   // The getter will wait until the setter has been called, and will return the
   // value passed during Set.
-  T& Get() {
+  T& Get() { return WaitWithTimeout(absl::Hours(1)); }
+  // The getter will wait with timeout until the setter has been called, and
+  // will return the value passed during Set.
+  T& WaitWithTimeout(absl::Duration d) {
     grpc_core::MutexLock lock(&mu_);
     if (!set_) {
-      cv_.Wait(&mu_);
+      cv_.WaitWithTimeout(&mu_, d);
     }
     return val_;
   }

data/src/core/lib/event_engine/socket_notifier.h

@@ -0,0 +1,55 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef GRPC_CORE_LIB_EVENT_ENGINE_SOCKET_NOTIFIER_H
+#define GRPC_CORE_LIB_EVENT_ENGINE_SOCKET_NOTIFIER_H
+
+#include <grpc/support/port_platform.h>
+
+#include "absl/status/status.h"
+
+#include <grpc/event_engine/event_engine.h>
+
+namespace grpc_event_engine {
+namespace experimental {
+
+// Generically wraps a socket/fd, and manages the registration of callbacks and
+// triggering of notifications on it.
+class SocketNotifier {
+ public:
+  virtual ~SocketNotifier() = default;
+  // Schedule on_read to be invoked when the underlying socket
+  // becomes readable.
+  // If the socket is already readable, the callback will be executed as soon as
+  // possible.
+  virtual void NotifyOnRead(EventEngine::Closure* on_read) = 0;
+  // Schedule on_write to be invoked when the underlying socket
+  // becomes writable.
+  // If the socket is already writable, the callback will be executed as soon as
+  // possible.
+  virtual void NotifyOnWrite(EventEngine::Closure* on_write) = 0;
+  // Set a readable event on the underlying socket.
+  virtual void SetReadable() = 0;
+  // Set a writable event on the underlying socket.
+  virtual void SetWritable() = 0;
+  // Shutdown a SocketNotifier. After this operation, NotifyXXX and SetXXX
+  // operations cannot be performed.
+  virtual void MaybeShutdown(absl::Status why) = 0;
+  // Returns true if the SocketNotifier has been shutdown.
+  virtual bool IsShutdown() = 0;
+};
+
+}  // namespace experimental
+}  // namespace grpc_event_engine
+
+#endif  // GRPC_CORE_LIB_EVENT_ENGINE_SOCKET_NOTIFIER_H

data/src/core/lib/event_engine/{iomgr_engine/thread_pool.cc → thread_pool.cc}

@@ -18,19 +18,21 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/lib/event_engine/iomgr_engine/thread_pool.h"
+#include "src/core/lib/event_engine/thread_pool.h"
+
+#include <utility>
 
 #include "src/core/lib/gprpp/thd.h"
 
 namespace grpc_event_engine {
-namespace iomgr_engine {
+namespace experimental {
 
 ThreadPool::Thread::Thread(ThreadPool* pool)
     : pool_(pool),
       thd_(
-          "iomgr_eventengine_pool",
+          "posix_eventengine_pool",
           [](void* th) { static_cast<ThreadPool::Thread*>(th)->ThreadFunc(); },
-          this) {
+          this, nullptr, grpc_core::Thread::Options().set_tracked(false)) {
   thd_.Start();
 }
 ThreadPool::Thread::~Thread() { thd_.Join(); }
@@ -43,8 +45,9 @@ void ThreadPool::Thread::ThreadFunc() {
   // Move ourselves to dead list
   pool_->dead_threads_.push_back(this);
 
-  if ((pool_->shutdown_) && (pool_->nthreads_ == 0)) {
-    pool_->shutdown_cv_.Signal();
+  if (pool_->nthreads_ == 0) {
+    if (pool_->forking_) pool_->fork_cv_.Signal();
+    if (pool_->shutdown_) pool_->shutdown_cv_.Signal();
   }
 }
 
@@ -52,7 +55,7 @@ void ThreadPool::ThreadFunc() {
   for (;;) {
     // Wait until work is available or we are shutting down.
     grpc_core::ReleasableMutexLock lock(&mu_);
-    if (!shutdown_ && callbacks_.empty()) {
+    if (!forking_ && !shutdown_ && callbacks_.empty()) {
       // If there are too many threads waiting, then quit this thread
       if (threads_waiting_ >= reserve_threads_) {
         break;
@@ -61,10 +64,12 @@ void ThreadPool::ThreadFunc() {
       cv_.Wait(&mu_);
       threads_waiting_--;
     }
+    // a fork could be initiated while the thread was waiting
+    if (forking_) return;
     // Drain callbacks before considering shutdown to ensure all work
     // gets completed.
     if (!callbacks_.empty()) {
-      auto cb = callbacks_.front();
+      auto cb = std::move(callbacks_.front());
       callbacks_.pop();
       lock.Release();
       cb();
@@ -78,9 +83,14 @@ ThreadPool::ThreadPool(int reserve_threads)
     : shutdown_(false),
       reserve_threads_(reserve_threads),
       nthreads_(0),
-      threads_waiting_(0) {
-  for (int i = 0; i < reserve_threads_; i++) {
-    grpc_core::MutexLock lock(&mu_);
+      threads_waiting_(0),
+      forking_(false) {
+  grpc_core::MutexLock lock(&mu_);
+  StartNThreadsLocked(reserve_threads_);
+}
+
+void ThreadPool::StartNThreadsLocked(int n) {
+  for (int i = 0; i < n; i++) {
     nthreads_++;
     new Thread(this);
   }
@@ -101,10 +111,13 @@ ThreadPool::~ThreadPool() {
   ReapThreads(&dead_threads_);
 }
 
-void ThreadPool::Add(const std::function<void()>& callback) {
+void ThreadPool::Add(absl::AnyInvocable<void()> callback) {
   grpc_core::MutexLock lock(&mu_);
   // Add works to the callbacks list
-  callbacks_.push(callback);
+  callbacks_.push(std::move(callback));
+  // Store the callback for later if we are forking.
+  // TODO(hork): should we block instead?
+  if (forking_) return;
   // Increase pool size or notify as needed
   if (threads_waiting_ == 0) {
     // Kick off a new thread
@@ -119,5 +132,27 @@ void ThreadPool::Add(const std::function<void()>& callback) {
   }
 }
 
-}  // namespace iomgr_engine
+void ThreadPool::PrepareFork() {
+  grpc_core::MutexLock lock(&mu_);
+  forking_ = true;
+  cv_.SignalAll();
+  while (nthreads_ != 0) {
+    fork_cv_.Wait(&mu_);
+  }
+  ReapThreads(&dead_threads_);
+}
+
+void ThreadPool::PostforkParent() {
+  grpc_core::MutexLock lock(&mu_);
+  forking_ = false;
+  StartNThreadsLocked(reserve_threads_);
+}
+
+void ThreadPool::PostforkChild() {
+  grpc_core::MutexLock lock(&mu_);
+  forking_ = false;
+  StartNThreadsLocked(reserve_threads_);
+}
+
+}  // namespace experimental
 }  // namespace grpc_event_engine

data/src/core/lib/event_engine/{iomgr_engine/thread_pool.h → thread_pool.h}

@@ -16,27 +16,35 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_THREAD_POOL_H
-#define GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_THREAD_POOL_H
+#ifndef GRPC_CORE_LIB_EVENT_ENGINE_THREAD_POOL_H
+#define GRPC_CORE_LIB_EVENT_ENGINE_THREAD_POOL_H
 
 #include <grpc/support/port_platform.h>
 
-#include <functional>
 #include <queue>
 #include <vector>
 
+#include "absl/base/thread_annotations.h"
+#include "absl/functional/any_invocable.h"
+
+#include "src/core/lib/event_engine/forkable.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/gprpp/thd.h"
 
 namespace grpc_event_engine {
-namespace iomgr_engine {
+namespace experimental {
 
-class ThreadPool final {
+class ThreadPool final : public grpc_event_engine::experimental::Forkable {
  public:
   explicit ThreadPool(int reserve_threads);
-  ~ThreadPool();
+  ~ThreadPool() override;
+
+  void Add(absl::AnyInvocable<void()> callback);
 
-  void Add(const std::function<void()>& callback);
+  // Forkable
+  void PrepareFork() override;
+  void PostforkParent() override;
+  void PostforkChild() override;
 
  private:
   class Thread {
@@ -51,20 +59,23 @@ class ThreadPool final {
   };
 
   void ThreadFunc();
+  void StartNThreadsLocked(int n) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&mu_);
   static void ReapThreads(std::vector<Thread*>* tlist);
 
   grpc_core::Mutex mu_;
   grpc_core::CondVar cv_;
   grpc_core::CondVar shutdown_cv_;
+  grpc_core::CondVar fork_cv_;
   bool shutdown_;
-  std::queue<std::function<void()>> callbacks_;
+  std::queue<absl::AnyInvocable<void()>> callbacks_;
   int reserve_threads_;
   int nthreads_;
   int threads_waiting_;
   std::vector<Thread*> dead_threads_;
+  bool forking_;
 };
 
-}  // namespace iomgr_engine
+}  // namespace experimental
 }  // namespace grpc_event_engine
 
-#endif  // GRPC_CORE_LIB_EVENT_ENGINE_IOMGR_ENGINE_THREAD_POOL_H
+#endif  // GRPC_CORE_LIB_EVENT_ENGINE_THREAD_POOL_H

data/src/core/lib/event_engine/utils.cc

@@ -0,0 +1,49 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/event_engine/utils.h"
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <chrono>
+
+#include "absl/strings/str_cat.h"
+
+#include <grpc/event_engine/event_engine.h>
+
+#include "src/core/lib/gprpp/time.h"
+
+namespace grpc_event_engine {
+namespace experimental {
+
+std::string HandleToString(EventEngine::TaskHandle handle) {
+  return absl::StrCat("{", handle.keys[0], ",", handle.keys[1], "}");
+}
+
+grpc_core::Timestamp ToTimestamp(grpc_core::Timestamp now,
+                                 EventEngine::Duration delta) {
+  return now +
+         std::max(grpc_core::Duration::Milliseconds(1),
+                  grpc_core::Duration::NanosecondsRoundUp(delta.count())) +
+         grpc_core::Duration::Milliseconds(1);
+}
+
+size_t Milliseconds(EventEngine::Duration d) {
+  return std::chrono::duration_cast<std::chrono::milliseconds>(d).count();
+}
+
+}  // namespace experimental
+}  // namespace grpc_event_engine

data/src/core/lib/event_engine/utils.h

@@ -0,0 +1,40 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef GRPC_CORE_LIB_EVENT_ENGINE_UTILS_H
+#define GRPC_CORE_LIB_EVENT_ENGINE_UTILS_H
+
+#include <grpc/support/port_platform.h>
+
+#include <stddef.h>
+
+#include <string>
+
+#include <grpc/event_engine/event_engine.h>
+
+#include "src/core/lib/gprpp/time.h"
+
+namespace grpc_event_engine {
+namespace experimental {
+
+std::string HandleToString(EventEngine::TaskHandle handle);
+
+grpc_core::Timestamp ToTimestamp(grpc_core::Timestamp now,
+                                 EventEngine::Duration delta);
+
+size_t Milliseconds(EventEngine::Duration d);
+
+}  // namespace experimental
+}  // namespace grpc_event_engine
+
+#endif  // GRPC_CORE_LIB_EVENT_ENGINE_UTILS_H

data/src/core/lib/event_engine/windows/iocp.cc

@@ -0,0 +1,149 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_WINDOWS
+
+#include <chrono>
+
+#include "absl/strings/str_format.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log_windows.h>
+
+#include "src/core/lib/event_engine/trace.h"
+#include "src/core/lib/event_engine/utils.h"
+#include "src/core/lib/event_engine/windows/iocp.h"
+#include "src/core/lib/event_engine/windows/win_socket.h"
+
+namespace grpc_event_engine {
+namespace experimental {
+
+IOCP::IOCP(Executor* executor) noexcept
+    : executor_(executor),
+      iocp_handle_(CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL,
+                                          (ULONG_PTR)NULL, 0)) {
+  GPR_ASSERT(iocp_handle_);
+  WSASocketFlagsInit();
+}
+
+// Shutdown must be called prior to deletion
+IOCP::~IOCP() {}
+
+WinSocket* IOCP::Watch(SOCKET socket) {
+  WinSocket* wrapped_socket = new WinSocket(socket, executor_);
+  HANDLE ret =
+      CreateIoCompletionPort(reinterpret_cast<HANDLE>(socket), iocp_handle_,
+                             reinterpret_cast<uintptr_t>(wrapped_socket), 0);
+  if (!ret) {
+    char* utf8_message = gpr_format_message(WSAGetLastError());
+    gpr_log(GPR_ERROR, "Unable to add socket to iocp: %s", utf8_message);
+    gpr_free(utf8_message);
+    __debugbreak();
+    abort();
+  }
+  GPR_ASSERT(ret == iocp_handle_);
+  return wrapped_socket;
+}
+
+void IOCP::Shutdown() {
+  while (outstanding_kicks_.load() > 0) {
+    Work(std::chrono::hours(42));
+  }
+  GPR_ASSERT(CloseHandle(iocp_handle_));
+}
+
+Poller::WorkResult IOCP::Work(EventEngine::Duration timeout) {
+  static const absl::Status kDeadlineExceeded = absl::DeadlineExceededError(
+      absl::StrFormat("IOCP::%p: Received no completions", this));
+  static const absl::Status kKicked =
+      absl::AbortedError(absl::StrFormat("IOCP::%p: Awoken from a kick", this));
+  DWORD bytes = 0;
+  ULONG_PTR completion_key;
+  LPOVERLAPPED overlapped;
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_event_engine_trace)) {
+    gpr_log(GPR_DEBUG, "IOCP::%p doing work", this);
+  }
+  BOOL success = GetQueuedCompletionStatus(
+      iocp_handle_, &bytes, &completion_key, &overlapped,
+      static_cast<DWORD>(Milliseconds(timeout)));
+  if (success == 0 && overlapped == NULL) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_event_engine_trace)) {
+      gpr_log(GPR_DEBUG, "IOCP::%p deadline exceeded", this);
+    }
+    return Poller::DeadlineExceeded{};
+  }
+  GPR_ASSERT(completion_key && overlapped);
+  if (overlapped == &kick_overlap_) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_event_engine_trace)) {
+      gpr_log(GPR_DEBUG, "IOCP::%p kicked", this);
+    }
+    outstanding_kicks_.fetch_sub(1);
+    if (completion_key == (ULONG_PTR)&kick_token_) {
+      return Poller::Kicked{};
+    }
+    gpr_log(GPR_ERROR, "Unknown custom completion key: %p", completion_key);
+    abort();
+  }
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_event_engine_trace)) {
+    gpr_log(GPR_DEBUG, "IOCP::%p got event on OVERLAPPED::%p", this,
+            overlapped);
+  }
+  WinSocket* socket = reinterpret_cast<WinSocket*>(completion_key);
+  // TODO(hork): move the following logic into the WinSocket impl.
+  WinSocket::OpState* info = socket->GetOpInfoForOverlapped(overlapped);
+  GPR_ASSERT(info != nullptr);
+  if (socket->IsShutdown()) {
+    info->SetError(WSAESHUTDOWN);
+  } else {
+    info->GetOverlappedResult();
+  }
+  if (info->closure() != nullptr) return Events{info->closure()};
+  // No callback registered. Set ready and return an empty set
+  info->SetReady();
+  return Events{};
+}
+
+void IOCP::Kick() {
+  outstanding_kicks_.fetch_add(1);
+  GPR_ASSERT(PostQueuedCompletionStatus(
+      iocp_handle_, 0, reinterpret_cast<ULONG_PTR>(&kick_token_),
+      &kick_overlap_));
+}
+
+DWORD IOCP::GetDefaultSocketFlags() {
+  static DWORD wsa_socket_flags = WSASocketFlagsInit();
+  return wsa_socket_flags;
+}
+
+DWORD IOCP::WSASocketFlagsInit() {
+  DWORD wsa_socket_flags = WSA_FLAG_OVERLAPPED;
+  /* WSA_FLAG_NO_HANDLE_INHERIT may be not supported on the older Windows
+     versions, see
+     https://msdn.microsoft.com/en-us/library/windows/desktop/ms742212(v=vs.85).aspx
+     for details. */
+  SOCKET sock = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, NULL, 0,
+                          wsa_socket_flags | WSA_FLAG_NO_HANDLE_INHERIT);
+  if (sock != INVALID_SOCKET) {
+    /* Windows 7, Windows 2008 R2 with SP1 or later */
+    wsa_socket_flags |= WSA_FLAG_NO_HANDLE_INHERIT;
+    closesocket(sock);
+  }
+  return wsa_socket_flags;
+}
+
+}  // namespace experimental
+}  // namespace grpc_event_engine
+
+#endif  // GPR_WINDOWS