grpc 1.44.0.pre2 → 1.46.3

data/src/core/lib/iomgr/polling_entity.h

@@ -23,6 +23,7 @@
 
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_set.h"
+#include "src/core/lib/promise/context.h"
 
 typedef enum grpc_pollset_tag {
   GRPC_POLLS_NONE,
@@ -65,4 +66,9 @@ void grpc_polling_entity_add_to_pollset_set(grpc_polling_entity* pollent,
 void grpc_polling_entity_del_from_pollset_set(grpc_polling_entity* pollent,
                                               grpc_pollset_set* pss_dst);
 
+namespace grpc_core {
+template <>
+struct ContextType<grpc_polling_entity> {};
+}  // namespace grpc_core
+
 #endif /* GRPC_CORE_LIB_IOMGR_POLLING_ENTITY_H */

data/src/core/lib/iomgr/pollset.cc

@@ -44,7 +44,7 @@ void grpc_pollset_destroy(grpc_pollset* pollset) {
 
 grpc_error_handle grpc_pollset_work(grpc_pollset* pollset,
                                     grpc_pollset_worker** worker,
-                                    grpc_millis deadline) {
+                                    grpc_core::Timestamp deadline) {
   return grpc_pollset_impl->work(pollset, worker, deadline);
 }
 

data/src/core/lib/iomgr/pollset.h

@@ -45,7 +45,7 @@ typedef struct grpc_pollset_vtable {
   void (*shutdown)(grpc_pollset* pollset, grpc_closure* closure);
   void (*destroy)(grpc_pollset* pollset);
   grpc_error_handle (*work)(grpc_pollset* pollset, grpc_pollset_worker** worker,
-                            grpc_millis deadline);
+                            grpc_core::Timestamp deadline);
   grpc_error_handle (*kick)(grpc_pollset* pollset,
                             grpc_pollset_worker* specific_worker);
   size_t (*pollset_size)(void);
@@ -86,9 +86,9 @@ void grpc_pollset_destroy(grpc_pollset* pollset);
    May call grpc_closure_list_run on grpc_closure_list, without holding the
    pollset
    lock */
-grpc_error_handle grpc_pollset_work(grpc_pollset* pollset,
-                                    grpc_pollset_worker** worker,
-                                    grpc_millis deadline) GRPC_MUST_USE_RESULT;
+grpc_error_handle grpc_pollset_work(
+    grpc_pollset* pollset, grpc_pollset_worker** worker,
+    grpc_core::Timestamp deadline) GRPC_MUST_USE_RESULT;
 
 /* Break one polling thread out of polling work for this pollset.
    If specific_worker is non-NULL, then kick that worker. */

data/src/core/lib/iomgr/pollset_windows.cc

@@ -108,7 +108,7 @@ static void pollset_destroy(grpc_pollset* pollset) {}
 
 static grpc_error_handle pollset_work(grpc_pollset* pollset,
                                       grpc_pollset_worker** worker_hdl,
-                                      grpc_millis deadline) {
+                                      grpc_core::Timestamp deadline) {
   grpc_pollset_worker worker;
   if (worker_hdl) *worker_hdl = &worker;
 
@@ -159,7 +159,7 @@ static grpc_error_handle pollset_work(grpc_pollset* pollset,
     added_worker = 1;
     while (!worker.kicked) {
       if (gpr_cv_wait(&worker.cv, &grpc_polling_mu,
-                      grpc_millis_to_timespec(deadline, GPR_CLOCK_REALTIME))) {
+                      deadline.as_timespec(GPR_CLOCK_REALTIME))) {
         grpc_core::ExecCtx::Get()->InvalidateNow();
         break;
       }

data/src/core/lib/iomgr/port.h

@@ -25,9 +25,7 @@
 #ifdef GPR_WINDOWS
 #define GRPC_ARES_RESOLVE_LOCALHOST_MANUALLY 1
 #endif
-#if defined(GRPC_CUSTOM_SOCKET)
-// Do Nothing
-#elif defined(GRPC_USE_EVENT_ENGINE)
+#if defined(GRPC_USE_EVENT_ENGINE)
 // Do Nothing
 #elif defined(GPR_WINDOWS)
 #define GRPC_WINSOCK_SOCKET 1
@@ -109,7 +107,6 @@
 #define GRPC_POSIX_SOCKET_ARES_EV_DRIVER 1
 #define GRPC_POSIX_SOCKET_EV 1
 #define GRPC_POSIX_SOCKET_EV_EPOLL1 1
-#define GRPC_POSIX_SOCKET_EV_EPOLLEX 1
 #define GRPC_POSIX_SOCKET_EV_POLL 1
 #define GRPC_POSIX_SOCKET_IF_NAMETOINDEX 1
 #define GRPC_POSIX_SOCKET_RESOLVE_ADDRESS 1
@@ -184,18 +181,16 @@
 #error "Platform not recognized"
 #endif
 
-#if defined(GRPC_POSIX_SOCKET) + defined(GRPC_WINSOCK_SOCKET) + \
-        defined(GRPC_CUSTOM_SOCKET) + defined(GRPC_CFSTREAM) +  \
-        defined(GRPC_USE_EVENT_ENGINE) !=                       \
+#if defined(GRPC_POSIX_SOCKET) + defined(GRPC_WINSOCK_SOCKET) +    \
+        defined(GRPC_CFSTREAM) + defined(GRPC_USE_EVENT_ENGINE) != \
     1
 #error \
-    "Must define exactly one of GRPC_POSIX_SOCKET, GRPC_WINSOCK_SOCKET, GRPC_CUSTOM_SOCKET, GRPC_CFSTREAM, GRPC_USE_EVENT_ENGINE"
+    "Must define exactly one of GRPC_POSIX_SOCKET, GRPC_WINSOCK_SOCKET, GRPC_CFSTREAM, GRPC_USE_EVENT_ENGINE"
 #endif
 
 #ifdef GRPC_POSIX_SOCKET
 #define GRPC_POSIX_SOCKET_ARES_EV_DRIVER 1
 #define GRPC_POSIX_SOCKET_EV 1
-#define GRPC_POSIX_SOCKET_EV_EPOLLEX 1
 #define GRPC_POSIX_SOCKET_EV_POLL 1
 #define GRPC_POSIX_SOCKET_EV_EPOLL1 1
 #define GRPC_POSIX_SOCKET_IF_NAMETOINDEX 1

data/src/core/lib/iomgr/resolve_address_posix.cc

@@ -121,7 +121,7 @@ NativeDNSResolver::ResolveNameBlocking(absl::string_view name,
     goto done;
   }
   if (port.empty()) {
-    if (default_port == nullptr) {
+    if (default_port.empty()) {
       err = grpc_error_set_str(
           GRPC_ERROR_CREATE_FROM_STATIC_STRING("no port in name"),
           GRPC_ERROR_STR_TARGET_ADDRESS, name);

data/src/core/lib/iomgr/sockaddr_utils_posix.cc

@@ -0,0 +1,62 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifdef GRPC_POSIX_SOCKET_UTILS_COMMON
+
+#include "src/core/lib/iomgr/socket_utils.h"
+#ifdef GRPC_LINUX_TCP_H
+#include <linux/tcp.h>
+#else
+#include <netinet/tcp.h>
+#endif
+#include <stdio.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <string>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+
+#include "src/core/lib/iomgr/sockaddr.h"
+
+uint16_t grpc_htons(uint16_t hostshort) { return htons(hostshort); }
+
+uint16_t grpc_ntohs(uint16_t netshort) { return ntohs(netshort); }
+
+uint32_t grpc_htonl(uint32_t hostlong) { return htonl(hostlong); }
+
+uint32_t grpc_ntohl(uint32_t netlong) { return ntohl(netlong); }
+
+int grpc_inet_pton(int af, const char* src, void* dst) {
+  return inet_pton(af, src, dst);
+}
+
+const char* grpc_inet_ntop(int af, const void* src, char* dst, size_t size) {
+  GPR_ASSERT(size <= (socklen_t)-1);
+  return inet_ntop(af, src, dst, static_cast<socklen_t>(size));
+}
+
+#endif

data/src/core/lib/iomgr/socket_utils_common_posix.cc

@@ -495,21 +495,4 @@ grpc_error_handle grpc_create_dualstack_socket_using_factory(
   return error_for_fd(*newfd, resolved_addr);
 }
 
-uint16_t grpc_htons(uint16_t hostshort) { return htons(hostshort); }
-
-uint16_t grpc_ntohs(uint16_t netshort) { return ntohs(netshort); }
-
-uint32_t grpc_htonl(uint32_t hostlong) { return htonl(hostlong); }
-
-uint32_t grpc_ntohl(uint32_t netlong) { return ntohl(netlong); }
-
-int grpc_inet_pton(int af, const char* src, void* dst) {
-  return inet_pton(af, src, dst);
-}
-
-const char* grpc_inet_ntop(int af, const void* src, char* dst, size_t size) {
-  GPR_ASSERT(size <= (socklen_t)-1);
-  return inet_ntop(af, src, dst, static_cast<socklen_t>(size));
-}
-
 #endif

data/src/core/lib/iomgr/tcp_client.cc

@@ -26,7 +26,7 @@ void grpc_tcp_client_connect(grpc_closure* on_connect, grpc_endpoint** endpoint,
                              grpc_pollset_set* interested_parties,
                              const grpc_channel_args* channel_args,
                              const grpc_resolved_address* addr,
-                             grpc_millis deadline) {
+                             grpc_core::Timestamp deadline) {
   grpc_tcp_client_impl->connect(on_connect, endpoint, interested_parties,
                                 channel_args, addr, deadline);
 }

data/src/core/lib/iomgr/tcp_client.h

@@ -33,7 +33,8 @@ typedef struct grpc_tcp_client_vtable {
   void (*connect)(grpc_closure* on_connect, grpc_endpoint** endpoint,
                   grpc_pollset_set* interested_parties,
                   const grpc_channel_args* channel_args,
-                  const grpc_resolved_address* addr, grpc_millis deadline);
+                  const grpc_resolved_address* addr,
+                  grpc_core::Timestamp deadline);
 } grpc_tcp_client_vtable;
 
 /* Asynchronously connect to an address (specified as (addr, len)), and call
@@ -45,7 +46,7 @@ void grpc_tcp_client_connect(grpc_closure* on_connect, grpc_endpoint** endpoint,
                              grpc_pollset_set* interested_parties,
                              const grpc_channel_args* channel_args,
                              const grpc_resolved_address* addr,
-                             grpc_millis deadline);
+                             grpc_core::Timestamp deadline);
 
 void grpc_tcp_client_global_init();
 

data/src/core/lib/iomgr/tcp_client_cfstream.cc

@@ -153,7 +153,7 @@ static void CFStreamClientConnect(grpc_closure* closure, grpc_endpoint** ep,
                                   grpc_pollset_set* interested_parties,
                                   const grpc_channel_args* channel_args,
                                   const grpc_resolved_address* resolved_addr,
-                                  grpc_millis deadline) {
+                                  grpc_core::Timestamp deadline) {
   CFStreamConnect* connect = new CFStreamConnect();
   connect->closure = closure;
   connect->endpoint = ep;

data/src/core/lib/iomgr/tcp_client_posix.cc

@@ -136,6 +136,7 @@ static void on_writable(void* acp, grpc_error_handle error) {
   int done;
   grpc_endpoint** ep = ac->ep;
   grpc_closure* closure = ac->closure;
+  std::string addr_str = ac->addr_str;
   grpc_fd* fd;
 
   (void)GRPC_ERROR_REF(error);
@@ -221,8 +222,7 @@ finish:
     std::string description =
         absl::StrCat("Failed to connect to remote host: ", str);
     error = grpc_error_set_str(error, GRPC_ERROR_STR_DESCRIPTION, description);
-    error =
-        grpc_error_set_str(error, GRPC_ERROR_STR_TARGET_ADDRESS, ac->addr_str);
+    error = grpc_error_set_str(error, GRPC_ERROR_STR_TARGET_ADDRESS, addr_str);
   }
   if (done) {
     // This is safe even outside the lock, because "done", the sentinel, is
@@ -270,7 +270,7 @@ grpc_error_handle grpc_tcp_client_prepare_fd(
 void grpc_tcp_client_create_from_prepared_fd(
     grpc_pollset_set* interested_parties, grpc_closure* closure, const int fd,
     const grpc_channel_args* channel_args, const grpc_resolved_address* addr,
-    grpc_millis deadline, grpc_endpoint** ep) {
+    grpc_core::Timestamp deadline, grpc_endpoint** ep) {
   int err;
   do {
     err = connect(fd, reinterpret_cast<const grpc_sockaddr*>(addr->addr),
@@ -325,7 +325,7 @@ static void tcp_connect(grpc_closure* closure, grpc_endpoint** ep,
                         grpc_pollset_set* interested_parties,
                         const grpc_channel_args* channel_args,
                         const grpc_resolved_address* addr,
-                        grpc_millis deadline) {
+                        grpc_core::Timestamp deadline) {
   grpc_resolved_address mapped_addr;
   int fd = -1;
   grpc_error_handle error;

data/src/core/lib/iomgr/tcp_client_posix.h

@@ -64,6 +64,6 @@ grpc_error_handle grpc_tcp_client_prepare_fd(
 void grpc_tcp_client_create_from_prepared_fd(
     grpc_pollset_set* interested_parties, grpc_closure* closure, const int fd,
     const grpc_channel_args* channel_args, const grpc_resolved_address* addr,
-    grpc_millis deadline, grpc_endpoint** ep);
+    grpc_core::Timestamp deadline, grpc_endpoint** ep);
 
 #endif /* GRPC_CORE_LIB_IOMGR_TCP_CLIENT_POSIX_H */

data/src/core/lib/iomgr/tcp_client_windows.cc

@@ -125,7 +125,7 @@ static void tcp_connect(grpc_closure* on_done, grpc_endpoint** endpoint,
                         grpc_pollset_set* interested_parties,
                         const grpc_channel_args* channel_args,
                         const grpc_resolved_address* addr,
-                        grpc_millis deadline) {
+                        grpc_core::Timestamp deadline) {
   SOCKET sock = INVALID_SOCKET;
   BOOL success;
   int status;

data/src/core/lib/iomgr/tcp_posix.cc

@@ -61,6 +61,7 @@
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/resource_quota/api.h"
 #include "src/core/lib/resource_quota/memory_quota.h"
+#include "src/core/lib/resource_quota/trace.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
 
@@ -365,6 +366,7 @@ struct grpc_tcp {
   /* Used by the endpoint read function to distinguish the very first read call
    * from the rest */
   bool is_first_read;
+  bool has_posted_reclaimer;
   double target_length;
   double bytes_read_this_round;
   grpc_core::RefCount refcount;
@@ -376,7 +378,8 @@ struct grpc_tcp {
   /* garbage after the last read */
   grpc_slice_buffer last_read_buffer;
 
-  grpc_slice_buffer* incoming_buffer;
+  absl::Mutex read_mu;
+  grpc_slice_buffer* incoming_buffer ABSL_GUARDED_BY(read_mu) = nullptr;
   int inq;          /* bytes pending on the socket from the last read. */
   bool inq_capable; /* cache whether kernel supports inq */
 
@@ -423,6 +426,9 @@ struct grpc_tcp {
                                       on errors anymore */
   TcpZerocopySendCtx tcp_zerocopy_send_ctx;
   TcpZerocopySendRecord* current_zerocopy_send = nullptr;
+
+  bool curr_read_completed;
+  int curr_min_read_chunk_size;
 };
 
 struct backup_poller {
@@ -461,7 +467,8 @@ static void run_poller(void* bp, grpc_error_handle /*error_ignored*/) {
     gpr_log(GPR_INFO, "BACKUP_POLLER:%p run", p);
   }
   gpr_mu_lock(p->pollset_mu);
-  grpc_millis deadline = grpc_core::ExecCtx::Get()->Now() + 10 * GPR_MS_PER_SEC;
+  grpc_core::Timestamp deadline =
+      grpc_core::ExecCtx::Get()->Now() + grpc_core::Duration::Seconds(10);
   GRPC_STATS_INC_TCP_BACKUP_POLLER_POLLS();
   GRPC_LOG_IF_ERROR(
       "backup_poller:pollset_work",
@@ -660,9 +667,35 @@ static void tcp_destroy(grpc_endpoint* ep) {
   TCP_UNREF(tcp, "destroy");
 }
 
-static void call_read_cb(grpc_tcp* tcp, grpc_error_handle error) {
-  grpc_closure* cb = tcp->read_cb;
+static void perform_reclamation(grpc_tcp* tcp)
+    ABSL_LOCKS_EXCLUDED(tcp->read_mu) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
+    gpr_log(GPR_INFO, "TCP: benign reclamation to free memory");
+  }
+  tcp->read_mu.Lock();
+  if (tcp->incoming_buffer != nullptr) {
+    grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
+  }
+  tcp->read_mu.Unlock();
+  tcp->has_posted_reclaimer = false;
+}
 
+static void maybe_post_reclaimer(grpc_tcp* tcp)
+    ABSL_EXCLUSIVE_LOCKS_REQUIRED(tcp->read_mu) {
+  if (!tcp->has_posted_reclaimer) {
+    tcp->has_posted_reclaimer = true;
+    tcp->memory_owner.PostReclaimer(
+        grpc_core::ReclamationPass::kBenign,
+        [tcp](absl::optional<grpc_core::ReclamationSweep> sweep) {
+          if (!sweep.has_value()) return;
+          perform_reclamation(tcp);
+        });
+  }
+}
+
+static void tcp_trace_read(grpc_tcp* tcp, grpc_error_handle error)
+    ABSL_EXCLUSIVE_LOCKS_REQUIRED(tcp->read_mu) {
+  grpc_closure* cb = tcp->read_cb;
   if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "TCP:%p call_cb %p %p:%p", tcp, cb, cb->cb, cb->cb_arg);
     size_t i;
@@ -677,15 +710,16 @@ static void call_read_cb(grpc_tcp* tcp, grpc_error_handle error) {
       }
     }
   }
-
-  tcp->read_cb = nullptr;
-  tcp->incoming_buffer = nullptr;
-  grpc_core::Closure::Run(DEBUG_LOCATION, cb, error);
 }
 
+/* Returns true if data available to read or error other than EAGAIN. */
 #define MAX_READ_IOVEC 4
-static void tcp_do_read(grpc_tcp* tcp) {
+static bool tcp_do_read(grpc_tcp* tcp, grpc_error_handle* error)
+    ABSL_EXCLUSIVE_LOCKS_REQUIRED(tcp->read_mu) {
   GPR_TIMER_SCOPE("tcp_do_read", 0);
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
+    gpr_log(GPR_INFO, "TCP:%p do_read", tcp);
+  }
   struct msghdr msg;
   struct iovec iov[MAX_READ_IOVEC];
   ssize_t read_bytes;
@@ -745,17 +779,16 @@ static void tcp_do_read(grpc_tcp* tcp) {
       /* NB: After calling call_read_cb a parallel call of the read handler may
        * be running. */
       if (errno == EAGAIN) {
+        tcp->curr_read_completed = true;
         finish_estimate(tcp);
         tcp->inq = 0;
-        /* We've consumed the edge, request a new one */
-        notify_on_read(tcp);
+        return false;
       } else {
+        tcp->curr_read_completed = false;
         grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
-        call_read_cb(tcp,
-                     tcp_annotate_error(GRPC_OS_ERROR(errno, "recvmsg"), tcp));
-        TCP_UNREF(tcp, "read");
+        *error = tcp_annotate_error(GRPC_OS_ERROR(errno, "recvmsg"), tcp);
+        return true;
       }
-      return;
     }
     if (read_bytes == 0) {
       /* 0 read size ==> end of stream
@@ -763,12 +796,11 @@ static void tcp_do_read(grpc_tcp* tcp) {
        * We may have read something, i.e., total_read_bytes > 0, but
        * since the connection is closed we will drop the data here, because we
        * can't call the callback multiple times. */
+      tcp->curr_read_completed = true;
       grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
-      call_read_cb(
-          tcp, tcp_annotate_error(
-                   GRPC_ERROR_CREATE_FROM_STATIC_STRING("Socket closed"), tcp));
-      TCP_UNREF(tcp, "read");
-      return;
+      *error = tcp_annotate_error(
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Socket closed"), tcp);
+      return true;
     }
 
     GRPC_STATS_INC_TCP_READ_SIZE(read_bytes);
@@ -821,17 +853,20 @@ static void tcp_do_read(grpc_tcp* tcp) {
     finish_estimate(tcp);
   }
 
+  // There may be more data to be read because recvmsg did not return EAGAIN.
+  tcp->curr_read_completed = false;
   GPR_DEBUG_ASSERT(total_read_bytes > 0);
   if (total_read_bytes < tcp->incoming_buffer->length) {
     grpc_slice_buffer_trim_end(tcp->incoming_buffer,
                                tcp->incoming_buffer->length - total_read_bytes,
                                &tcp->last_read_buffer);
   }
-  call_read_cb(tcp, GRPC_ERROR_NONE);
-  TCP_UNREF(tcp, "read");
+  *error = GRPC_ERROR_NONE;
+  return true;
 }
 
-static void tcp_continue_read(grpc_tcp* tcp) {
+static void maybe_make_read_slices(grpc_tcp* tcp)
+    ABSL_EXCLUSIVE_LOCKS_REQUIRED(tcp->read_mu) {
   if (tcp->incoming_buffer->length == 0 &&
       tcp->incoming_buffer->count < MAX_READ_IOVEC) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
@@ -844,17 +879,24 @@ static void tcp_continue_read(grpc_tcp* tcp) {
     int target_length = static_cast<int>(tcp->target_length);
     int extra_wanted =
         target_length - static_cast<int>(tcp->incoming_buffer->length);
+    if (tcp->curr_read_completed) {
+      // Set it to false again to start the next block of reads
+      tcp->curr_read_completed = false;
+      // Reset curr_min_read_chunk_size for the next block of reads
+      tcp->curr_min_read_chunk_size = tcp->min_read_chunk_size;
+    } else {
+      // Last read is not completed yet. Double the last min read chunk size.
+      tcp->curr_min_read_chunk_size =
+          std::min(2 * tcp->curr_min_read_chunk_size, tcp->max_read_chunk_size);
+    }
     grpc_slice_buffer_add_indexed(
         tcp->incoming_buffer,
         tcp->memory_owner.MakeSlice(grpc_core::MemoryRequest(
-            tcp->min_read_chunk_size,
-            grpc_core::Clamp(extra_wanted, tcp->min_read_chunk_size,
+            tcp->curr_min_read_chunk_size,
+            grpc_core::Clamp(extra_wanted, tcp->curr_min_read_chunk_size,
                              tcp->max_read_chunk_size))));
+    maybe_post_reclaimer(tcp);
   }
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
-    gpr_log(GPR_INFO, "TCP:%p do_read", tcp);
-  }
-  tcp_do_read(tcp);
 }
 
 static void tcp_handle_read(void* arg /* grpc_tcp */, grpc_error_handle error) {
@@ -863,15 +905,28 @@ static void tcp_handle_read(void* arg /* grpc_tcp */, grpc_error_handle error) {
     gpr_log(GPR_INFO, "TCP:%p got_read: %s", tcp,
             grpc_error_std_string(error).c_str());
   }
-
-  if (GPR_UNLIKELY(error != GRPC_ERROR_NONE)) {
+  tcp->read_mu.Lock();
+  grpc_error_handle tcp_read_error;
+  if (GPR_LIKELY(error == GRPC_ERROR_NONE)) {
+    maybe_make_read_slices(tcp);
+    if (!tcp_do_read(tcp, &tcp_read_error)) {
+      /* We've consumed the edge, request a new one */
+      tcp->read_mu.Unlock();
+      notify_on_read(tcp);
+      return;
+    }
+    tcp_trace_read(tcp, tcp_read_error);
+  } else {
+    tcp_read_error = GRPC_ERROR_REF(error);
     grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
     grpc_slice_buffer_reset_and_unref_internal(&tcp->last_read_buffer);
-    call_read_cb(tcp, GRPC_ERROR_REF(error));
-    TCP_UNREF(tcp, "read");
-  } else {
-    tcp_continue_read(tcp);
   }
+  grpc_closure* cb = tcp->read_cb;
+  tcp->read_cb = nullptr;
+  tcp->incoming_buffer = nullptr;
+  tcp->read_mu.Unlock();
+  grpc_core::Closure::Run(DEBUG_LOCATION, cb, tcp_read_error);
+  TCP_UNREF(tcp, "read");
 }
 
 static void tcp_read(grpc_endpoint* ep, grpc_slice_buffer* incoming_buffer,
@@ -879,9 +934,11 @@ static void tcp_read(grpc_endpoint* ep, grpc_slice_buffer* incoming_buffer,
   grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
   GPR_ASSERT(tcp->read_cb == nullptr);
   tcp->read_cb = cb;
+  tcp->read_mu.Lock();
   tcp->incoming_buffer = incoming_buffer;
   grpc_slice_buffer_reset_and_unref_internal(incoming_buffer);
   grpc_slice_buffer_swap(incoming_buffer, &tcp->last_read_buffer);
+  tcp->read_mu.Unlock();
   TCP_REF(tcp, "read");
   if (tcp->is_first_read) {
     /* Endpoint read called for the very first time. Register read callback with
@@ -1740,17 +1797,19 @@ grpc_endpoint* grpc_tcp_create(grpc_fd* em_fd,
   tcp->current_zerocopy_send = nullptr;
   tcp->release_fd_cb = nullptr;
   tcp->release_fd = nullptr;
-  tcp->incoming_buffer = nullptr;
   tcp->target_length = static_cast<double>(tcp_read_chunk_size);
   tcp->min_read_chunk_size = tcp_min_read_chunk_size;
   tcp->max_read_chunk_size = tcp_max_read_chunk_size;
   tcp->bytes_read_this_round = 0;
   /* Will be set to false by the very first endpoint read function */
   tcp->is_first_read = true;
+  tcp->has_posted_reclaimer = false;
   tcp->bytes_counter = -1;
   tcp->socket_ts_enabled = false;
   tcp->ts_capable = true;
   tcp->outgoing_buffer_arg = nullptr;
+  tcp->curr_read_completed = true;
+  tcp->curr_min_read_chunk_size = tcp->min_read_chunk_size;
   if (tcp_tx_zerocopy_enabled && !tcp->tcp_zerocopy_send_ctx.memory_limited()) {
 #ifdef GRPC_LINUX_ERRQUEUE
     const int enable = 1;

data/src/core/lib/iomgr/tcp_server_posix.cc

@@ -29,6 +29,7 @@
 
 #include <errno.h>
 #include <fcntl.h>
+#include <inttypes.h>
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include <string.h>
@@ -61,6 +62,8 @@
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
 #include "src/core/lib/resource_quota/api.h"
 
+static std::atomic<int64_t> num_dropped_connections{0};
+
 static grpc_error_handle tcp_server_create(grpc_closure* shutdown_complete,
                                            const grpc_channel_args* args,
                                            grpc_tcp_server** server) {
@@ -201,27 +204,33 @@ static void on_read(void* arg, grpc_error_handle err) {
        strip off the ::ffff:0.0.0.0/96 prefix first. */
     int fd = grpc_accept4(sp->fd, &addr, 1, 1);
     if (fd < 0) {
-      switch (errno) {
-        case EINTR:
-          continue;
-        case EAGAIN:
-          grpc_fd_notify_on_read(sp->emfd, &sp->read_closure);
-          return;
-        default:
-          gpr_mu_lock(&sp->server->mu);
-          if (!sp->server->shutdown_listeners) {
-            gpr_log(GPR_ERROR, "Failed accept4: %s", strerror(errno));
-          } else {
-            /* if we have shutdown listeners, accept4 could fail, and we
-               needn't notify users */
-          }
-          gpr_mu_unlock(&sp->server->mu);
-          goto error;
+      if (errno == EINTR) {
+        continue;
+      } else if (errno == EAGAIN || errno == ECONNABORTED ||
+                 errno == EWOULDBLOCK) {
+        grpc_fd_notify_on_read(sp->emfd, &sp->read_closure);
+        return;
+      } else {
+        gpr_mu_lock(&sp->server->mu);
+        if (!sp->server->shutdown_listeners) {
+          gpr_log(GPR_ERROR, "Failed accept4: %s", strerror(errno));
+        } else {
+          /* if we have shutdown listeners, accept4 could fail, and we
+             needn't notify users */
+        }
+        gpr_mu_unlock(&sp->server->mu);
+        goto error;
       }
     }
 
     if (sp->server->memory_quota->IsMemoryPressureHigh()) {
-      gpr_log(GPR_INFO, "Drop incoming connection: high memory pressure");
+      int64_t dropped_connections_count = ++num_dropped_connections;
+      if (dropped_connections_count % 1000 == 0) {
+        gpr_log(GPR_INFO,
+                "Dropped >= %" PRId64
+                " new connection attempts due to high memory pressure",
+                dropped_connections_count);
+      }
       close(fd);
       continue;
     }