grpc 1.54.2 → 1.56.0.pre3

data/src/core/lib/iomgr/ev_epoll1_linux.cc

@@ -61,6 +61,7 @@
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 
 static grpc_wakeup_fd global_wakeup_fd;
+static bool g_is_shutdown = true;
 
 //******************************************************************************
 // Singleton epoll set related fields
@@ -1239,6 +1240,7 @@ static void shutdown_engine(void) {
     gpr_mu_destroy(&fork_fd_list_mu);
     grpc_core::Fork::SetResetChildPollingEngineFunc(nullptr);
   }
+  g_is_shutdown = true;
 }
 
 static bool init_epoll1_linux();
@@ -1278,10 +1280,13 @@ const grpc_event_engine_vtable grpc_ev_epoll1_posix = {
 
     is_any_background_poller_thread,
     /* name = */ "epoll1",
-    /* check_engine_available = */ [](bool) { return init_epoll1_linux(); },
-    /* init_engine = */ []() {},
+    /* check_engine_available = */
+    [](bool) { return init_epoll1_linux(); },
+    /* init_engine = */
+    []() { GPR_ASSERT(init_epoll1_linux()); },
     shutdown_background_closure,
-    /* shutdown_engine = */ []() {},
+    /* shutdown_engine = */
+    []() { shutdown_engine(); },
     add_closure_to_background_poller,
 
     fd_set_pre_allocated,
@@ -1306,6 +1311,7 @@ static void reset_event_manager_on_fork() {
 // Create epoll_fd (epoll_set_init() takes care of that) to make sure epoll
 // support is available
 static bool init_epoll1_linux() {
+  if (!g_is_shutdown) return true;
   if (!grpc_has_wakeup_fd()) {
     gpr_log(GPR_ERROR, "Skipping epoll1 because of no wakeup fd.");
     return false;
@@ -1328,6 +1334,7 @@ static bool init_epoll1_linux() {
     grpc_core::Fork::SetResetChildPollingEngineFunc(
         reset_event_manager_on_fork);
   }
+  g_is_shutdown = false;
   return true;
 }
 

data/src/core/lib/iomgr/event_engine_shims/endpoint.cc

@@ -256,6 +256,15 @@ class EventEngineEndpointWrapper {
     }
   }
 
+  bool CanTrackErrors() {
+    if (EventEngineSupportsFd()) {
+      return reinterpret_cast<PosixEndpointWithFdSupport*>(endpoint_.get())
+          ->CanTrackErrors();
+    } else {
+      return false;
+    }
+  }
+
  private:
   void OnShutdownInternal() {
     {
@@ -378,7 +387,12 @@ int EndpointGetFd(grpc_endpoint* ep) {
   return eeep->wrapper->Fd();
 }
 
-bool EndpointCanTrackErr(grpc_endpoint* /* ep */) { return false; }
+bool EndpointCanTrackErr(grpc_endpoint* ep) {
+  auto* eeep =
+      reinterpret_cast<EventEngineEndpointWrapper::grpc_event_engine_endpoint*>(
+          ep);
+  return eeep->wrapper->CanTrackErrors();
+}
 
 grpc_endpoint_vtable grpc_event_engine_endpoint_vtable = {
     EndpointRead,

data/src/core/lib/iomgr/iocp_windows.cc

@@ -43,6 +43,7 @@ static ULONG g_iocp_kick_token;
 static OVERLAPPED g_iocp_custom_overlap;
 
 static gpr_atm g_custom_events = 0;
+static gpr_atm g_pending_socket_shutdowns = 0;
 
 static HANDLE g_iocp;
 
@@ -90,6 +91,7 @@ grpc_iocp_work_status grpc_iocp_work(grpc_core::Timestamp deadline) {
   } else {
     abort();
   }
+  gpr_mu_lock(&socket->state_mu);
   if (socket->shutdown_called) {
     info->bytes_transferred = 0;
     info->wsa_error = WSA_OPERATION_ABORTED;
@@ -100,7 +102,11 @@ grpc_iocp_work_status grpc_iocp_work(grpc_core::Timestamp deadline) {
     info->wsa_error = success ? 0 : WSAGetLastError();
   }
   GPR_ASSERT(overlapped == &info->overlapped);
-  grpc_socket_become_ready(socket, info);
+  bool should_destroy = grpc_socket_become_ready(socket, info);
+  gpr_mu_unlock(&socket->state_mu);
+  if (should_destroy) {
+    grpc_winsocket_finish(socket);
+  }
   return GRPC_IOCP_WORK_WORK;
 }
 
@@ -122,11 +128,13 @@ void grpc_iocp_kick(void) {
 void grpc_iocp_flush(void) {
   grpc_core::ExecCtx exec_ctx;
   grpc_iocp_work_status work_status;
-
+  // This method is called during grpc_shutdown. We make the loop
+  // spin until any pending socket shutdowns are complete.
   do {
     work_status = grpc_iocp_work(grpc_core::Timestamp::InfPast());
   } while (work_status == GRPC_IOCP_WORK_KICK ||
-           grpc_core::ExecCtx::Get()->Flush());
+           grpc_core::ExecCtx::Get()->Flush() ||
+           gpr_atm_acq_load(&g_pending_socket_shutdowns) != 0);
 }
 
 void grpc_iocp_shutdown(void) {
@@ -155,4 +163,17 @@ void grpc_iocp_add_socket(grpc_winsocket* socket) {
   GPR_ASSERT(ret == g_iocp);
 }
 
+void grpc_iocp_register_socket_shutdown_socket_locked(grpc_winsocket* socket) {
+  if (!socket->shutdown_registered) {
+    socket->shutdown_registered = true;
+    gpr_atm_full_fetch_add(&g_pending_socket_shutdowns, 1);
+  }
+}
+
+void grpc_iocp_finish_socket_shutdown(grpc_winsocket* socket) {
+  if (socket->shutdown_registered) {
+    gpr_atm_full_fetch_add(&g_pending_socket_shutdowns, -1);
+  }
+}
+
 #endif  // GRPC_WINSOCK_SOCKET

data/src/core/lib/iomgr/iocp_windows.h

@@ -43,6 +43,17 @@ void grpc_iocp_flush(void);
 void grpc_iocp_shutdown(void);
 void grpc_iocp_add_socket(grpc_winsocket*);
 
+// Register that this socket has started shutting down.
+// This prevents gRPC from completing its own shutdown until this socket's
+// shutdown is finished. IOCP must continue doing work until all such sockets
+// have finished shutting down. The socket's state_mu must be locked.
+void grpc_iocp_register_socket_shutdown_socket_locked(grpc_winsocket* socket);
+
+// Mark that this socket has finished shutting down.
+// The socket's state lock does not need to be held since this function is only
+// called once the socket is ready to be destroyed.
+void grpc_iocp_finish_socket_shutdown(grpc_winsocket* socket);
+
 #endif
 
 #endif  // GRPC_SRC_CORE_LIB_IOMGR_IOCP_WINDOWS_H

data/src/core/lib/iomgr/iomgr_posix_cfstream.cc

@@ -73,7 +73,7 @@ static bool apple_iomgr_platform_is_any_background_poller_thread(void) {
 }
 
 static bool apple_iomgr_platform_add_closure_to_background_poller(
-    grpc_closure* closure, grpc_error_handle error) {
+    grpc_closure* /*closure*/, grpc_error_handle /*error*/) {
   return false;
 }
 

data/src/core/lib/iomgr/socket_utils_common_posix.cc

@@ -357,8 +357,10 @@ grpc_error_handle grpc_set_socket_tcp_user_timeout(
           return absl::OkStatus();
         }
         if (newval != timeout) {
-          // Do not fail on failing to set TCP_USER_TIMEOUT for now.
-          gpr_log(GPR_ERROR, "Failed to set TCP_USER_TIMEOUT");
+          gpr_log(GPR_INFO,
+                  "Setting TCP_USER_TIMEOUT to value %d ms. Actual "
+                  "TCP_USER_TIMEOUT value is %d ms",
+                  timeout, newval);
           return absl::OkStatus();
         }
       }

data/src/core/lib/iomgr/socket_utils_posix.cc

@@ -77,6 +77,9 @@ PosixTcpOptions TcpOptionsFromEndpointConfig(const EndpointConfig& config) {
   options.tcp_tx_zerocopy_max_simultaneous_sends =
       AdjustValue(PosixTcpOptions::kDefaultMaxSends, 0, INT_MAX,
                   config.GetInt(GRPC_ARG_TCP_TX_ZEROCOPY_MAX_SIMULT_SENDS));
+  options.tcp_receive_buffer_size =
+      AdjustValue(PosixTcpOptions::kReadBufferSizeUnset, 0, INT_MAX,
+                  config.GetInt(GRPC_ARG_TCP_RECEIVE_BUFFER_SIZE));
   options.tcp_tx_zero_copy_enabled =
       (AdjustValue(PosixTcpOptions::kZerocpTxEnabledDefault, 0, 1,
                    config.GetInt(GRPC_ARG_TCP_TX_ZEROCOPY_ENABLED)) != 0);

data/src/core/lib/iomgr/socket_utils_posix.h

@@ -49,11 +49,14 @@ struct PosixTcpOptions {
   static constexpr int kMaxChunkSize = 32 * 1024 * 1024;
   static constexpr int kDefaultMaxSends = 4;
   static constexpr size_t kDefaultSendBytesThreshold = 16 * 1024;
+  // Let the system decide the proper buffer size.
+  static constexpr int kReadBufferSizeUnset = -1;
   int tcp_read_chunk_size = kDefaultReadChunkSize;
   int tcp_min_read_chunk_size = kDefaultMinReadChunksize;
   int tcp_max_read_chunk_size = kDefaultMaxReadChunksize;
   int tcp_tx_zerocopy_send_bytes_threshold = kDefaultSendBytesThreshold;
   int tcp_tx_zerocopy_max_simultaneous_sends = kDefaultMaxSends;
+  int tcp_receive_buffer_size = kReadBufferSizeUnset;
   bool tcp_tx_zero_copy_enabled = kZerocpTxEnabledDefault;
   int keep_alive_time_ms = 0;
   int keep_alive_timeout_ms = 0;

data/src/core/lib/iomgr/socket_windows.cc

@@ -76,6 +76,51 @@ void grpc_winsocket_shutdown(grpc_winsocket* winsocket) {
     return;
   }
   winsocket->shutdown_called = true;
+  bool register_shutdown = false;
+  // If there is already a scheduled read closure, run it immediately. This
+  // follows the same semantics applied to posix endpoint which also runs any
+  // already registered closure immediately in the event of a shutdown.
+  if (winsocket->read_info.closure && !winsocket->read_info.has_pending_iocp) {
+    winsocket->read_info.bytes_transferred = 0;
+    winsocket->read_info.wsa_error = WSA_OPERATION_ABORTED;
+    grpc_core::ExecCtx::Run(DEBUG_LOCATION, winsocket->read_info.closure,
+                            absl::OkStatus());
+    // Note that while the read_info.closure closure is run, it is not set to
+    // NULL here. This ensures that the socket cannot get deleted yet until any
+    // pending I/O operations are flushed by the thread executing
+    // grpc_iocp_work. We set read_info.closure_already_executed_at_shutdown to
+    // true so that when the pending read I/O operations are flushed, the
+    // associated closure is not executed in the grpc_socket_became_ready
+    // function.
+    winsocket->read_info.closure_already_executed_at_shutdown = true;
+    register_shutdown = true;
+  }
+
+  // If there is already a scheduled write closure, run it immediately. This
+  // follows the same semantics applied to posix endpoint which also runs any
+  // already registered closure immediately in the event of a shutdown.
+  if (winsocket->write_info.closure &&
+      !winsocket->write_info.has_pending_iocp) {
+    winsocket->write_info.bytes_transferred = 0;
+    winsocket->write_info.wsa_error = WSA_OPERATION_ABORTED;
+    grpc_core::ExecCtx::Run(DEBUG_LOCATION, winsocket->write_info.closure,
+                            absl::OkStatus());
+    // Note that while the write_info.closure closure is run, it is not set to
+    // NULL here. This ensures that the socket cannot get deleted yet until any
+    // pending I/O operations are flushed by the thread executing
+    // grpc_iocp_work. We set
+    // write_info.closure.closure_already_executed_at_shutdown to true so that
+    // when the pending write I/O operations are flushed, the associated closure
+    // is not executed in the grpc_socket_became_ready function.
+    winsocket->write_info.closure_already_executed_at_shutdown = true;
+    register_shutdown = true;
+  }
+
+  if (register_shutdown) {
+    // Instruct gRPC to avoid completing any shutdowns until this socket is
+    // cleaned up.
+    grpc_iocp_register_socket_shutdown_socket_locked(winsocket);
+  }
   gpr_mu_unlock(&winsocket->state_mu);
 
   status = WSAIoctl(winsocket->socket, SIO_GET_EXTENSION_FUNCTION_POINTER,
@@ -90,6 +135,8 @@ void grpc_winsocket_shutdown(grpc_winsocket* winsocket) {
             utf8_message);
     gpr_free(utf8_message);
   }
+  // Calling closesocket triggers invocation of any pending I/O operations with
+  // ABORTED status.
   closesocket(winsocket->socket);
 }
 
@@ -105,13 +152,20 @@ static bool check_destroyable(grpc_winsocket* winsocket) {
          winsocket->read_info.closure == NULL;
 }
 
+void grpc_winsocket_finish(grpc_winsocket* winsocket) {
+  grpc_iocp_finish_socket_shutdown(winsocket);
+  destroy(winsocket);
+}
+
 void grpc_winsocket_destroy(grpc_winsocket* winsocket) {
   gpr_mu_lock(&winsocket->state_mu);
   GPR_ASSERT(!winsocket->destroy_called);
   winsocket->destroy_called = true;
   bool should_destroy = check_destroyable(winsocket);
   gpr_mu_unlock(&winsocket->state_mu);
-  if (should_destroy) destroy(winsocket);
+  if (should_destroy) {
+    grpc_winsocket_finish(winsocket);
+  }
 }
 
 // Calling notify_on_read or write means either of two things:
@@ -140,19 +194,19 @@ void grpc_socket_notify_on_read(grpc_winsocket* socket, grpc_closure* closure) {
   socket_notify_on_iocp(socket, closure, &socket->read_info);
 }
 
-void grpc_socket_become_ready(grpc_winsocket* socket,
+bool grpc_socket_become_ready(grpc_winsocket* socket,
                               grpc_winsocket_callback_info* info) {
   GPR_ASSERT(!info->has_pending_iocp);
-  gpr_mu_lock(&socket->state_mu);
   if (info->closure) {
-    grpc_core::ExecCtx::Run(DEBUG_LOCATION, info->closure, absl::OkStatus());
+    // Only run the closure once at shutdown.
+    if (!info->closure_already_executed_at_shutdown) {
+      grpc_core::ExecCtx::Run(DEBUG_LOCATION, info->closure, absl::OkStatus());
+    }
     info->closure = NULL;
   } else {
     info->has_pending_iocp = 1;
   }
-  bool should_destroy = check_destroyable(socket);
-  gpr_mu_unlock(&socket->state_mu);
-  if (should_destroy) destroy(socket);
+  return check_destroyable(socket);
 }
 
 static gpr_once g_probe_ipv6_once = GPR_ONCE_INIT;

data/src/core/lib/iomgr/socket_windows.h

@@ -61,6 +61,9 @@ typedef struct grpc_winsocket_callback_info {
   // The results of the overlapped operation.
   DWORD bytes_transferred;
   int wsa_error;
+  // Tracks whether the final closure has already been run when the socket is
+  // shut down. This allows closures to be run immediately upon socket shutdown.
+  bool closure_already_executed_at_shutdown = false;
 } grpc_winsocket_callback_info;
 
 // This is a wrapper to a Windows socket. A socket can have one outstanding
@@ -81,6 +84,7 @@ typedef struct grpc_winsocket {
 
   gpr_mu state_mu;
   bool shutdown_called;
+  bool shutdown_registered;
 
   // You can't add the same socket twice to the same IO Completion Port.
   // This prevents that.
@@ -109,8 +113,8 @@ void grpc_socket_notify_on_write(grpc_winsocket* winsocket,
 void grpc_socket_notify_on_read(grpc_winsocket* winsocket,
                                 grpc_closure* closure);
 
-void grpc_socket_become_ready(grpc_winsocket* winsocket,
-                              grpc_winsocket_callback_info* ci);
+bool grpc_socket_become_ready(grpc_winsocket* socket,
+                              grpc_winsocket_callback_info* info);
 
 // Returns true if this system can create AF_INET6 sockets bound to ::1.
 // The value is probed once, and cached for the life of the process.
@@ -120,6 +124,9 @@ void grpc_wsa_socket_flags_init();
 
 DWORD grpc_get_default_wsa_socket_flags();
 
+// Final cleanup operations on the socket prior to deletion.
+void grpc_winsocket_finish(grpc_winsocket*);
+
 #endif
 
 #endif  // GRPC_SRC_CORE_LIB_IOMGR_SOCKET_WINDOWS_H

data/src/core/lib/iomgr/tcp_client_cfstream.cc

@@ -33,6 +33,7 @@
 #include <grpc/support/sync.h>
 
 #include "src/core/lib/address_utils/sockaddr_utils.h"
+#include "src/core/lib/event_engine/shim.h"
 #include "src/core/lib/gprpp/crash.h"
 #include "src/core/lib/gprpp/host_port.h"
 #include "src/core/lib/iomgr/cfstream_handle.h"
@@ -40,6 +41,7 @@
 #include "src/core/lib/iomgr/endpoint_cfstream.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/error_cfstream.h"
+#include "src/core/lib/iomgr/event_engine_shims/tcp_client.h"
 #include "src/core/lib/iomgr/tcp_client.h"
 #include "src/core/lib/iomgr/timer.h"
 
@@ -149,9 +151,14 @@ static void ParseResolvedAddress(const grpc_resolved_address* addr,
 
 static int64_t CFStreamClientConnect(
     grpc_closure* closure, grpc_endpoint** ep,
-    grpc_pollset_set* interested_parties,
-    const grpc_event_engine::experimental::EndpointConfig& /*config*/,
+    grpc_pollset_set* /*interested_parties*/,
+    const grpc_event_engine::experimental::EndpointConfig& config,
     const grpc_resolved_address* resolved_addr, grpc_core::Timestamp deadline) {
+  if (grpc_event_engine::experimental::UseEventEngineClient()) {
+    return grpc_event_engine::experimental::event_engine_tcp_client_connect(
+        closure, ep, config, resolved_addr, deadline);
+  }
+
   auto addr_uri = grpc_sockaddr_to_uri(resolved_addr);
   if (!addr_uri.ok()) {
     grpc_error_handle error = GRPC_ERROR_CREATE(addr_uri.status().ToString());
@@ -198,7 +205,11 @@ static int64_t CFStreamClientConnect(
   return 0;
 }
 
-static bool CFStreamClientCancelConnect(int64_t /*connection_handle*/) {
+static bool CFStreamClientCancelConnect(int64_t connection_handle) {
+  if (grpc_event_engine::experimental::UseEventEngineClient()) {
+    return grpc_event_engine::experimental::
+        event_engine_tcp_client_cancel_connect(connection_handle);
+  }
   return false;
 }
 

data/src/core/lib/iomgr/tcp_client_posix.cc

@@ -107,6 +107,10 @@ static grpc_error_handle prepare_socket(
   if (!err.ok()) goto error;
   err = grpc_set_socket_cloexec(fd, 1);
   if (!err.ok()) goto error;
+  if (options.tcp_receive_buffer_size != options.kReadBufferSizeUnset) {
+    err = grpc_set_socket_rcvbuf(fd, options.tcp_receive_buffer_size);
+    if (!err.ok()) goto error;
+  }
   if (!grpc_is_unix_socket(addr)) {
     err = grpc_set_socket_low_latency(fd, 1);
     if (!err.ok()) goto error;