grpc 1.18.0 → 1.22.0

data/src/core/lib/iomgr/tcp_posix.cc

@@ -22,12 +22,12 @@
 
 #ifdef GRPC_POSIX_SOCKET_TCP
 
-#include "src/core/lib/iomgr/network_status_tracker.h"
 #include "src/core/lib/iomgr/tcp_posix.h"
 
 #include <errno.h>
 #include <limits.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -35,6 +35,7 @@
 #include <sys/socket.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <algorithm>
 
 #include <grpc/slice.h>
 #include <grpc/support/alloc.h>
@@ -55,6 +56,15 @@
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
 
+#ifndef SOL_TCP
+#define SOL_TCP IPPROTO_TCP
+#endif
+
+#ifndef TCP_INQ
+#define TCP_INQ 36
+#define TCP_CM_INQ TCP_INQ
+#endif
+
 #ifdef GRPC_HAVE_MSG_NOSIGNAL
 #define SENDMSG_FLAGS MSG_NOSIGNAL
 #else
@@ -89,8 +99,11 @@ struct grpc_tcp {
   grpc_slice_buffer last_read_buffer;
 
   grpc_slice_buffer* incoming_buffer;
+  int inq;          /* bytes pending on the socket from the last read. */
+  bool inq_capable; /* cache whether kernel supports inq */
+
   grpc_slice_buffer* outgoing_buffer;
-  /** byte within outgoing_buffer->slices[0] to write next */
+  /* byte within outgoing_buffer->slices[0] to write next */
   size_t outgoing_byte_idx;
 
   grpc_closure* read_cb;
@@ -127,9 +140,8 @@ struct grpc_tcp {
   bool socket_ts_enabled; /* True if timestamping options are set on the socket
                            */
   bool ts_capable;        /* Cache whether we can set timestamping options */
-  gpr_atm
-      stop_error_notification; /* Set to 1 if we do not want to be notified on
-                                  errors anymore */
+  gpr_atm stop_error_notification; /* Set to 1 if we do not want to be notified
+                                      on errors anymore */
 };
 
 struct backup_poller {
@@ -151,7 +163,7 @@ static void tcp_drop_uncovered_then_handle_write(void* arg /* grpc_tcp */,
 
 static void done_poller(void* bp, grpc_error* error_ignored) {
   backup_poller* p = static_cast<backup_poller*>(bp);
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "BACKUP_POLLER:%p destroy", p);
   }
   grpc_pollset_destroy(BACKUP_POLLER_POLLSET(p));
@@ -160,7 +172,7 @@ static void done_poller(void* bp, grpc_error* error_ignored) {
 
 static void run_poller(void* bp, grpc_error* error_ignored) {
   backup_poller* p = static_cast<backup_poller*>(bp);
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "BACKUP_POLLER:%p run", p);
   }
   gpr_mu_lock(p->pollset_mu);
@@ -176,18 +188,18 @@ static void run_poller(void* bp, grpc_error* error_ignored) {
       gpr_atm_full_cas(&g_uncovered_notifications_pending, 1, 0)) {
     gpr_mu_lock(p->pollset_mu);
     bool cas_ok = gpr_atm_full_cas(&g_backup_poller, (gpr_atm)p, 0);
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       gpr_log(GPR_INFO, "BACKUP_POLLER:%p done cas_ok=%d", p, cas_ok);
     }
     gpr_mu_unlock(p->pollset_mu);
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       gpr_log(GPR_INFO, "BACKUP_POLLER:%p shutdown", p);
     }
     grpc_pollset_shutdown(BACKUP_POLLER_POLLSET(p),
                           GRPC_CLOSURE_INIT(&p->run_poller, done_poller, p,
                                             grpc_schedule_on_exec_ctx));
   } else {
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       gpr_log(GPR_INFO, "BACKUP_POLLER:%p reschedule", p);
     }
     GRPC_CLOSURE_SCHED(&p->run_poller, GRPC_ERROR_NONE);
@@ -197,8 +209,8 @@ static void run_poller(void* bp, grpc_error* error_ignored) {
 static void drop_uncovered(grpc_tcp* tcp) {
   backup_poller* p = (backup_poller*)gpr_atm_acq_load(&g_backup_poller);
   gpr_atm old_count =
-      gpr_atm_no_barrier_fetch_add(&g_uncovered_notifications_pending, -1);
-  if (grpc_tcp_trace.enabled()) {
+      gpr_atm_full_fetch_add(&g_uncovered_notifications_pending, -1);
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "BACKUP_POLLER:%p uncover cnt %d->%d", p,
             static_cast<int>(old_count), static_cast<int>(old_count) - 1);
   }
@@ -216,7 +228,7 @@ static void cover_self(grpc_tcp* tcp) {
   backup_poller* p;
   gpr_atm old_count =
       gpr_atm_no_barrier_fetch_add(&g_uncovered_notifications_pending, 2);
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "BACKUP_POLLER: cover cnt %d->%d",
             static_cast<int>(old_count), 2 + static_cast<int>(old_count));
   }
@@ -224,22 +236,22 @@ static void cover_self(grpc_tcp* tcp) {
     GRPC_STATS_INC_TCP_BACKUP_POLLERS_CREATED();
     p = static_cast<backup_poller*>(
         gpr_zalloc(sizeof(*p) + grpc_pollset_size()));
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       gpr_log(GPR_INFO, "BACKUP_POLLER:%p create", p);
     }
     grpc_pollset_init(BACKUP_POLLER_POLLSET(p), &p->pollset_mu);
     gpr_atm_rel_store(&g_backup_poller, (gpr_atm)p);
-    GRPC_CLOSURE_SCHED(
-        GRPC_CLOSURE_INIT(&p->run_poller, run_poller, p,
-                          grpc_executor_scheduler(GRPC_EXECUTOR_LONG)),
-        GRPC_ERROR_NONE);
+    GRPC_CLOSURE_SCHED(GRPC_CLOSURE_INIT(&p->run_poller, run_poller, p,
+                                         grpc_core::Executor::Scheduler(
+                                             grpc_core::ExecutorJobType::LONG)),
+                       GRPC_ERROR_NONE);
   } else {
     while ((p = (backup_poller*)gpr_atm_acq_load(&g_backup_poller)) ==
            nullptr) {
       // spin waiting for backup poller
     }
   }
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "BACKUP_POLLER:%p add %p", p, tcp);
   }
   grpc_pollset_add_fd(BACKUP_POLLER_POLLSET(p), tcp->em_fd);
@@ -249,34 +261,24 @@ static void cover_self(grpc_tcp* tcp) {
 }
 
 static void notify_on_read(grpc_tcp* tcp) {
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "TCP:%p notify_on_read", tcp);
   }
-  GRPC_CLOSURE_INIT(&tcp->read_done_closure, tcp_handle_read, tcp,
-                    grpc_schedule_on_exec_ctx);
   grpc_fd_notify_on_read(tcp->em_fd, &tcp->read_done_closure);
 }
 
 static void notify_on_write(grpc_tcp* tcp) {
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "TCP:%p notify_on_write", tcp);
   }
-  if (grpc_event_engine_run_in_background()) {
-    // If there is a polling engine always running in the background, there is
-    // no need to run the backup poller.
-    GRPC_CLOSURE_INIT(&tcp->write_done_closure, tcp_handle_write, tcp,
-                      grpc_schedule_on_exec_ctx);
-  } else {
+  if (!grpc_event_engine_run_in_background()) {
     cover_self(tcp);
-    GRPC_CLOSURE_INIT(&tcp->write_done_closure,
-                      tcp_drop_uncovered_then_handle_write, tcp,
-                      grpc_schedule_on_exec_ctx);
   }
   grpc_fd_notify_on_write(tcp->em_fd, &tcp->write_done_closure);
 }
 
 static void tcp_drop_uncovered_then_handle_write(void* arg, grpc_error* error) {
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "TCP:%p got_write: %s", arg, grpc_error_string(error));
   }
   drop_uncovered(static_cast<grpc_tcp*>(arg));
@@ -345,6 +347,13 @@ static void tcp_free(grpc_tcp* tcp) {
   grpc_slice_buffer_destroy_internal(&tcp->last_read_buffer);
   grpc_resource_user_unref(tcp->resource_user);
   gpr_free(tcp->peer_string);
+  /* The lock is not really necessary here, since all refs have been released */
+  gpr_mu_lock(&tcp->tb_mu);
+  grpc_core::TracedBuffer::Shutdown(
+      &tcp->tb_head, tcp->outgoing_buffer_arg,
+      GRPC_ERROR_CREATE_FROM_STATIC_STRING("endpoint destroyed"));
+  gpr_mu_unlock(&tcp->tb_mu);
+  tcp->outgoing_buffer_arg = nullptr;
   gpr_mu_destroy(&tcp->tb_mu);
   gpr_free(tcp);
 }
@@ -354,7 +363,7 @@ static void tcp_free(grpc_tcp* tcp) {
 #define TCP_REF(tcp, reason) tcp_ref((tcp), (reason), __FILE__, __LINE__)
 static void tcp_unref(grpc_tcp* tcp, const char* reason, const char* file,
                       int line) {
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_atm val = gpr_atm_no_barrier_load(&tcp->refcount.count);
     gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
             "TCP unref %p : %s %" PRIdPTR " -> %" PRIdPTR, tcp, reason, val,
@@ -367,7 +376,7 @@ static void tcp_unref(grpc_tcp* tcp, const char* reason, const char* file,
 
 static void tcp_ref(grpc_tcp* tcp, const char* reason, const char* file,
                     int line) {
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_atm val = gpr_atm_no_barrier_load(&tcp->refcount.count);
     gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
             "TCP   ref %p : %s %" PRIdPTR " -> %" PRIdPTR, tcp, reason, val,
@@ -388,16 +397,9 @@ static void tcp_ref(grpc_tcp* tcp) { gpr_ref(&tcp->refcount); }
 #endif
 
 static void tcp_destroy(grpc_endpoint* ep) {
-  grpc_network_status_unregister_endpoint(ep);
   grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
   grpc_slice_buffer_reset_and_unref_internal(&tcp->last_read_buffer);
   if (grpc_event_engine_can_track_errors()) {
-    gpr_mu_lock(&tcp->tb_mu);
-    grpc_core::TracedBuffer::Shutdown(
-        &tcp->tb_head, tcp->outgoing_buffer_arg,
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("endpoint destroyed"));
-    gpr_mu_unlock(&tcp->tb_mu);
-    tcp->outgoing_buffer_arg = nullptr;
     gpr_atm_no_barrier_store(&tcp->stop_error_notification, true);
     grpc_fd_set_error(tcp->em_fd);
   }
@@ -407,17 +409,19 @@ static void tcp_destroy(grpc_endpoint* ep) {
 static void call_read_cb(grpc_tcp* tcp, grpc_error* error) {
   grpc_closure* cb = tcp->read_cb;
 
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "TCP:%p call_cb %p %p:%p", tcp, cb, cb->cb, cb->cb_arg);
     size_t i;
     const char* str = grpc_error_string(error);
-    gpr_log(GPR_INFO, "read: error=%s", str);
-
-    for (i = 0; i < tcp->incoming_buffer->count; i++) {
-      char* dump = grpc_dump_slice(tcp->incoming_buffer->slices[i],
-                                   GPR_DUMP_HEX | GPR_DUMP_ASCII);
-      gpr_log(GPR_INFO, "READ %p (peer=%s): %s", tcp, tcp->peer_string, dump);
-      gpr_free(dump);
+    gpr_log(GPR_INFO, "READ %p (peer=%s) error=%s", tcp, tcp->peer_string, str);
+
+    if (gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) {
+      for (i = 0; i < tcp->incoming_buffer->count; i++) {
+        char* dump = grpc_dump_slice(tcp->incoming_buffer->slices[i],
+                                     GPR_DUMP_HEX | GPR_DUMP_ASCII);
+        gpr_log(GPR_DEBUG, "DATA: %s", dump);
+        gpr_free(dump);
+      }
     }
   }
 
@@ -431,74 +435,145 @@ static void tcp_do_read(grpc_tcp* tcp) {
   GPR_TIMER_SCOPE("tcp_do_read", 0);
   struct msghdr msg;
   struct iovec iov[MAX_READ_IOVEC];
+  char cmsgbuf[24 /*CMSG_SPACE(sizeof(int))*/];
   ssize_t read_bytes;
-  size_t i;
+  size_t total_read_bytes = 0;
 
-  GPR_ASSERT(tcp->incoming_buffer->count <= MAX_READ_IOVEC);
-
-  for (i = 0; i < tcp->incoming_buffer->count; i++) {
+  size_t iov_len =
+      std::min<size_t>(MAX_READ_IOVEC, tcp->incoming_buffer->count);
+  for (size_t i = 0; i < iov_len; i++) {
     iov[i].iov_base = GRPC_SLICE_START_PTR(tcp->incoming_buffer->slices[i]);
     iov[i].iov_len = GRPC_SLICE_LENGTH(tcp->incoming_buffer->slices[i]);
   }
 
-  msg.msg_name = nullptr;
-  msg.msg_namelen = 0;
-  msg.msg_iov = iov;
-  msg.msg_iovlen = static_cast<msg_iovlen_type>(tcp->incoming_buffer->count);
-  msg.msg_control = nullptr;
-  msg.msg_controllen = 0;
-  msg.msg_flags = 0;
-
-  GRPC_STATS_INC_TCP_READ_OFFER(tcp->incoming_buffer->length);
-  GRPC_STATS_INC_TCP_READ_OFFER_IOV_SIZE(tcp->incoming_buffer->count);
-
   do {
-    GPR_TIMER_SCOPE("recvmsg", 0);
-    GRPC_STATS_INC_SYSCALL_READ();
-    read_bytes = recvmsg(tcp->fd, &msg, 0);
-  } while (read_bytes < 0 && errno == EINTR);
-
-  if (read_bytes < 0) {
-    /* NB: After calling call_read_cb a parallel call of the read handler may
-     * be running. */
-    if (errno == EAGAIN) {
-      finish_estimate(tcp);
-      /* We've consumed the edge, request a new one */
-      notify_on_read(tcp);
+    /* Assume there is something on the queue. If we receive TCP_INQ from
+     * kernel, we will update this value, otherwise, we have to assume there is
+     * always something to read until we get EAGAIN. */
+    tcp->inq = 1;
+
+    msg.msg_name = nullptr;
+    msg.msg_namelen = 0;
+    msg.msg_iov = iov;
+    msg.msg_iovlen = static_cast<msg_iovlen_type>(iov_len);
+    if (tcp->inq_capable) {
+      msg.msg_control = cmsgbuf;
+      msg.msg_controllen = sizeof(cmsgbuf);
     } else {
+      msg.msg_control = nullptr;
+      msg.msg_controllen = 0;
+    }
+    msg.msg_flags = 0;
+
+    GRPC_STATS_INC_TCP_READ_OFFER(tcp->incoming_buffer->length);
+    GRPC_STATS_INC_TCP_READ_OFFER_IOV_SIZE(tcp->incoming_buffer->count);
+
+    do {
+      GPR_TIMER_SCOPE("recvmsg", 0);
+      GRPC_STATS_INC_SYSCALL_READ();
+      read_bytes = recvmsg(tcp->fd, &msg, 0);
+    } while (read_bytes < 0 && errno == EINTR);
+
+    /* We have read something in previous reads. We need to deliver those
+     * bytes to the upper layer. */
+    if (read_bytes <= 0 && total_read_bytes > 0) {
+      tcp->inq = 1;
+      break;
+    }
+
+    if (read_bytes < 0) {
+      /* NB: After calling call_read_cb a parallel call of the read handler may
+       * be running. */
+      if (errno == EAGAIN) {
+        finish_estimate(tcp);
+        tcp->inq = 0;
+        /* We've consumed the edge, request a new one */
+        notify_on_read(tcp);
+      } else {
+        grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
+        call_read_cb(tcp,
+                     tcp_annotate_error(GRPC_OS_ERROR(errno, "recvmsg"), tcp));
+        TCP_UNREF(tcp, "read");
+      }
+      return;
+    }
+    if (read_bytes == 0) {
+      /* 0 read size ==> end of stream
+       *
+       * We may have read something, i.e., total_read_bytes > 0, but
+       * since the connection is closed we will drop the data here, because we
+       * can't call the callback multiple times. */
       grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
-      call_read_cb(tcp,
-                   tcp_annotate_error(GRPC_OS_ERROR(errno, "recvmsg"), tcp));
+      call_read_cb(
+          tcp, tcp_annotate_error(
+                   GRPC_ERROR_CREATE_FROM_STATIC_STRING("Socket closed"), tcp));
       TCP_UNREF(tcp, "read");
+      return;
     }
-  } else if (read_bytes == 0) {
-    /* 0 read size ==> end of stream */
-    grpc_slice_buffer_reset_and_unref_internal(tcp->incoming_buffer);
-    call_read_cb(
-        tcp, tcp_annotate_error(
-                 GRPC_ERROR_CREATE_FROM_STATIC_STRING("Socket closed"), tcp));
-    TCP_UNREF(tcp, "read");
-  } else {
+
     GRPC_STATS_INC_TCP_READ_SIZE(read_bytes);
     add_to_estimate(tcp, static_cast<size_t>(read_bytes));
-    GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length);
-    if (static_cast<size_t>(read_bytes) == tcp->incoming_buffer->length) {
-      finish_estimate(tcp);
-    } else if (static_cast<size_t>(read_bytes) < tcp->incoming_buffer->length) {
-      grpc_slice_buffer_trim_end(
-          tcp->incoming_buffer,
-          tcp->incoming_buffer->length - static_cast<size_t>(read_bytes),
-          &tcp->last_read_buffer);
+    GPR_DEBUG_ASSERT((size_t)read_bytes <=
+                     tcp->incoming_buffer->length - total_read_bytes);
+
+#ifdef GRPC_HAVE_TCP_INQ
+    if (tcp->inq_capable) {
+      GPR_DEBUG_ASSERT(!(msg.msg_flags & MSG_CTRUNC));
+      struct cmsghdr* cmsg = CMSG_FIRSTHDR(&msg);
+      for (; cmsg != nullptr; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+        if (cmsg->cmsg_level == SOL_TCP && cmsg->cmsg_type == TCP_CM_INQ &&
+            cmsg->cmsg_len == CMSG_LEN(sizeof(int))) {
+          tcp->inq = *reinterpret_cast<int*>(CMSG_DATA(cmsg));
+        }
+      }
     }
-    GPR_ASSERT((size_t)read_bytes == tcp->incoming_buffer->length);
-    call_read_cb(tcp, GRPC_ERROR_NONE);
-    TCP_UNREF(tcp, "read");
+#endif /* GRPC_HAVE_TCP_INQ */
+
+    total_read_bytes += read_bytes;
+    if (tcp->inq == 0 || total_read_bytes == tcp->incoming_buffer->length) {
+      /* We have filled incoming_buffer, and we cannot read any more. */
+      break;
+    }
+
+    /* We had a partial read, and still have space to read more data.
+     * So, adjust IOVs and try to read more. */
+    size_t remaining = read_bytes;
+    size_t j = 0;
+    for (size_t i = 0; i < iov_len; i++) {
+      if (remaining >= iov[i].iov_len) {
+        remaining -= iov[i].iov_len;
+        continue;
+      }
+      if (remaining > 0) {
+        iov[j].iov_base = static_cast<char*>(iov[i].iov_base) + remaining;
+        iov[j].iov_len = iov[i].iov_len - remaining;
+        remaining = 0;
+      } else {
+        iov[j].iov_base = iov[i].iov_base;
+        iov[j].iov_len = iov[i].iov_len;
+      }
+      ++j;
+    }
+    iov_len = j;
+  } while (true);
+
+  if (tcp->inq == 0) {
+    finish_estimate(tcp);
+  }
+
+  GPR_DEBUG_ASSERT(total_read_bytes > 0);
+  if (total_read_bytes < tcp->incoming_buffer->length) {
+    grpc_slice_buffer_trim_end(tcp->incoming_buffer,
+                               tcp->incoming_buffer->length - total_read_bytes,
+                               &tcp->last_read_buffer);
   }
+  call_read_cb(tcp, GRPC_ERROR_NONE);
+  TCP_UNREF(tcp, "read");
 }
 
 static void tcp_read_allocation_done(void* tcpp, grpc_error* error) {
   grpc_tcp* tcp = static_cast<grpc_tcp*>(tcpp);
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "TCP:%p read_allocation_done: %s", tcp,
             grpc_error_string(error));
   }
@@ -514,15 +589,16 @@ static void tcp_read_allocation_done(void* tcpp, grpc_error* error) {
 
 static void tcp_continue_read(grpc_tcp* tcp) {
   size_t target_read_size = get_target_read_size(tcp);
-  if (tcp->incoming_buffer->length < target_read_size / 2 &&
+  /* Wait for allocation only when there is no buffer left. */
+  if (tcp->incoming_buffer->length == 0 &&
       tcp->incoming_buffer->count < MAX_READ_IOVEC) {
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       gpr_log(GPR_INFO, "TCP:%p alloc_slices", tcp);
     }
     grpc_resource_user_alloc_slices(&tcp->slice_allocator, target_read_size, 1,
                                     tcp->incoming_buffer);
   } else {
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       gpr_log(GPR_INFO, "TCP:%p do_read", tcp);
     }
     tcp_do_read(tcp);
@@ -531,7 +607,7 @@ static void tcp_continue_read(grpc_tcp* tcp) {
 
 static void tcp_handle_read(void* arg /* grpc_tcp */, grpc_error* error) {
   grpc_tcp* tcp = static_cast<grpc_tcp*>(arg);
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "TCP:%p got_read: %s", tcp, grpc_error_string(error));
   }
 
@@ -546,7 +622,7 @@ static void tcp_handle_read(void* arg /* grpc_tcp */, grpc_error* error) {
 }
 
 static void tcp_read(grpc_endpoint* ep, grpc_slice_buffer* incoming_buffer,
-                     grpc_closure* cb) {
+                     grpc_closure* cb, bool urgent) {
   grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
   GPR_ASSERT(tcp->read_cb == nullptr);
   tcp->read_cb = cb;
@@ -559,6 +635,11 @@ static void tcp_read(grpc_endpoint* ep, grpc_slice_buffer* incoming_buffer,
      * the polling engine */
     tcp->is_first_read = false;
     notify_on_read(tcp);
+  } else if (!urgent && tcp->inq == 0) {
+    /* Upper layer asked to read more but we know there is no pending data
+     * to read from previous reads. So, wait for POLLIN.
+     */
+    notify_on_read(tcp);
   } else {
     /* Not the first time. We may or may not have more bytes available. In any
      * case call tcp->read_done_closure (i.e tcp_handle_read()) which does the
@@ -596,6 +677,7 @@ static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg,
 static void tcp_handle_error(void* arg /* grpc_tcp */, grpc_error* error);
 
 #ifdef GRPC_LINUX_ERRQUEUE
+
 static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg,
                                       size_t sending_length,
                                       ssize_t* sent_length) {
@@ -603,8 +685,7 @@ static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg,
     uint32_t opt = grpc_core::kTimestampingSocketOptions;
     if (setsockopt(tcp->fd, SOL_SOCKET, SO_TIMESTAMPING,
                    static_cast<void*>(&opt), sizeof(opt)) != 0) {
-      grpc_slice_buffer_reset_and_unref_internal(tcp->outgoing_buffer);
-      if (grpc_tcp_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
         gpr_log(GPR_ERROR, "Failed to set timestamping options on the socket.");
       }
       return false;
@@ -634,7 +715,7 @@ static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg,
     gpr_mu_lock(&tcp->tb_mu);
     grpc_core::TracedBuffer::AddNewEntry(
         &tcp->tb_head, static_cast<uint32_t>(tcp->bytes_counter + length),
-        tcp->outgoing_buffer_arg);
+        tcp->fd, tcp->outgoing_buffer_arg);
     gpr_mu_unlock(&tcp->tb_mu);
     tcp->outgoing_buffer_arg = nullptr;
   }
@@ -651,17 +732,31 @@ static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg,
 struct cmsghdr* process_timestamp(grpc_tcp* tcp, msghdr* msg,
                                   struct cmsghdr* cmsg) {
   auto next_cmsg = CMSG_NXTHDR(msg, cmsg);
+  cmsghdr* opt_stats = nullptr;
   if (next_cmsg == nullptr) {
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       gpr_log(GPR_ERROR, "Received timestamp without extended error");
     }
     return cmsg;
   }
 
+  /* Check if next_cmsg is an OPT_STATS msg */
+  if (next_cmsg->cmsg_level == SOL_SOCKET &&
+      next_cmsg->cmsg_type == SCM_TIMESTAMPING_OPT_STATS) {
+    opt_stats = next_cmsg;
+    next_cmsg = CMSG_NXTHDR(msg, opt_stats);
+    if (next_cmsg == nullptr) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
+        gpr_log(GPR_ERROR, "Received timestamp without extended error");
+      }
+      return opt_stats;
+    }
+  }
+
   if (!(next_cmsg->cmsg_level == SOL_IP || next_cmsg->cmsg_level == SOL_IPV6) ||
       !(next_cmsg->cmsg_type == IP_RECVERR ||
         next_cmsg->cmsg_type == IPV6_RECVERR)) {
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       gpr_log(GPR_ERROR, "Unexpected control message");
     }
     return cmsg;
@@ -679,7 +774,8 @@ struct cmsghdr* process_timestamp(grpc_tcp* tcp, msghdr* msg,
    * to protect the traced buffer list. A lock free list might be better. Using
    * a simple mutex for now. */
   gpr_mu_lock(&tcp->tb_mu);
-  grpc_core::TracedBuffer::ProcessTimestamp(&tcp->tb_head, serr, tss);
+  grpc_core::TracedBuffer::ProcessTimestamp(&tcp->tb_head, serr, opt_stats,
+                                            tss);
   gpr_mu_unlock(&tcp->tb_mu);
   return next_cmsg;
 }
@@ -699,9 +795,15 @@ static void process_errors(grpc_tcp* tcp) {
     msg.msg_iovlen = 0;
     msg.msg_flags = 0;
 
+    /* Allocate enough space so we don't need to keep increasing this as size
+     * of OPT_STATS increase */
+    constexpr size_t cmsg_alloc_space =
+        CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) +
+        CMSG_SPACE(sizeof(sock_extended_err) + sizeof(sockaddr_in)) +
+        CMSG_SPACE(32 * NLA_ALIGN(NLA_HDRLEN + sizeof(uint64_t)));
+    /* Allocate aligned space for cmsgs received along with timestamps */
     union {
-      char rbuf[1024 /*CMSG_SPACE(sizeof(scm_timestamping)) +
-                CMSG_SPACE(sizeof(sock_extended_err) + sizeof(sockaddr_in))*/];
+      char rbuf[cmsg_alloc_space];
       struct cmsghdr align;
     } aligned_buf;
     memset(&aligned_buf, 0, sizeof(aligned_buf));
@@ -721,10 +823,8 @@ static void process_errors(grpc_tcp* tcp) {
     if (r == -1) {
       return;
     }
-    if (grpc_tcp_trace.enabled()) {
-      if ((msg.msg_flags & MSG_CTRUNC) == 1) {
-        gpr_log(GPR_INFO, "Error message was truncated.");
-      }
+    if ((msg.msg_flags & MSG_CTRUNC) != 0) {
+      gpr_log(GPR_ERROR, "Error message was truncated.");
     }
 
     if (msg.msg_controllen == 0) {
@@ -738,7 +838,7 @@ static void process_errors(grpc_tcp* tcp) {
           cmsg->cmsg_type != SCM_TIMESTAMPING) {
         /* Got a control message that is not a timestamp. Don't know how to
          * handle this. */
-        if (grpc_tcp_trace.enabled()) {
+        if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
           gpr_log(GPR_INFO,
                   "unknown control message cmsg_level:%d cmsg_type:%d",
                   cmsg->cmsg_level, cmsg->cmsg_type);
@@ -756,7 +856,7 @@ static void process_errors(grpc_tcp* tcp) {
 
 static void tcp_handle_error(void* arg /* grpc_tcp */, grpc_error* error) {
   grpc_tcp* tcp = static_cast<grpc_tcp*>(arg);
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     gpr_log(GPR_INFO, "TCP:%p got_error: %s", tcp, grpc_error_string(error));
   }
 
@@ -775,8 +875,6 @@ static void tcp_handle_error(void* arg /* grpc_tcp */, grpc_error* error) {
    * ready. */
   grpc_fd_set_readable(tcp->em_fd);
   grpc_fd_set_writable(tcp->em_fd);
-  GRPC_CLOSURE_INIT(&tcp->error_closure, tcp_handle_error, tcp,
-                    grpc_schedule_on_exec_ctx);
   grpc_fd_notify_on_error(tcp->em_fd, &tcp->error_closure);
 }
 
@@ -881,8 +979,7 @@ static bool tcp_flush(grpc_tcp* tcp, grpc_error** error) {
         // unref all and forget about all slices that have been written to this
         // point
         for (size_t idx = 0; idx < unwind_slice_idx; ++idx) {
-          grpc_slice_unref_internal(
-              grpc_slice_buffer_take_first(tcp->outgoing_buffer));
+          grpc_slice_buffer_remove_first(tcp->outgoing_buffer);
         }
         return false;
       } else if (errno == EPIPE) {
@@ -935,14 +1032,14 @@ static void tcp_handle_write(void* arg /* grpc_tcp */, grpc_error* error) {
   }
 
   if (!tcp_flush(tcp, &error)) {
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       gpr_log(GPR_INFO, "write: delayed");
     }
     notify_on_write(tcp);
   } else {
     cb = tcp->write_cb;
     tcp->write_cb = nullptr;
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       const char* str = grpc_error_string(error);
       gpr_log(GPR_INFO, "write: %s", str);
     }
@@ -957,14 +1054,17 @@ static void tcp_write(grpc_endpoint* ep, grpc_slice_buffer* buf,
   grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
   grpc_error* error = GRPC_ERROR_NONE;
 
-  if (grpc_tcp_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
     size_t i;
 
     for (i = 0; i < buf->count; i++) {
-      char* data =
-          grpc_dump_slice(buf->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII);
-      gpr_log(GPR_INFO, "WRITE %p (peer=%s): %s", tcp, tcp->peer_string, data);
-      gpr_free(data);
+      gpr_log(GPR_INFO, "WRITE %p (peer=%s)", tcp, tcp->peer_string);
+      if (gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) {
+        char* data =
+            grpc_dump_slice(buf->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII);
+        gpr_log(GPR_DEBUG, "DATA: %s", data);
+        gpr_free(data);
+      }
     }
   }
 
@@ -989,12 +1089,12 @@ static void tcp_write(grpc_endpoint* ep, grpc_slice_buffer* buf,
   if (!tcp_flush(tcp, &error)) {
     TCP_REF(tcp, "write");
     tcp->write_cb = cb;
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       gpr_log(GPR_INFO, "write: delayed");
     }
     notify_on_write(tcp);
   } else {
-    if (grpc_tcp_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_tcp_trace)) {
       const char* str = grpc_error_string(error);
       gpr_log(GPR_INFO, "write: %s", str);
     }
@@ -1131,11 +1231,34 @@ grpc_endpoint* grpc_tcp_create(grpc_fd* em_fd,
   tcp->resource_user = grpc_resource_user_create(resource_quota, peer_string);
   grpc_resource_user_slice_allocator_init(
       &tcp->slice_allocator, tcp->resource_user, tcp_read_allocation_done, tcp);
-  /* Tell network status tracker about new endpoint */
-  grpc_network_status_register_endpoint(&tcp->base);
   grpc_resource_quota_unref_internal(resource_quota);
   gpr_mu_init(&tcp->tb_mu);
   tcp->tb_head = nullptr;
+  GRPC_CLOSURE_INIT(&tcp->read_done_closure, tcp_handle_read, tcp,
+                    grpc_schedule_on_exec_ctx);
+  if (grpc_event_engine_run_in_background()) {
+    // If there is a polling engine always running in the background, there is
+    // no need to run the backup poller.
+    GRPC_CLOSURE_INIT(&tcp->write_done_closure, tcp_handle_write, tcp,
+                      grpc_schedule_on_exec_ctx);
+  } else {
+    GRPC_CLOSURE_INIT(&tcp->write_done_closure,
+                      tcp_drop_uncovered_then_handle_write, tcp,
+                      grpc_schedule_on_exec_ctx);
+  }
+  /* Always assume there is something on the queue to read. */
+  tcp->inq = 1;
+#ifdef GRPC_HAVE_TCP_INQ
+  int one = 1;
+  if (setsockopt(tcp->fd, SOL_TCP, TCP_INQ, &one, sizeof(one)) == 0) {
+    tcp->inq_capable = true;
+  } else {
+    gpr_log(GPR_DEBUG, "cannot set inq fd=%d errno=%d", tcp->fd, errno);
+    tcp->inq_capable = false;
+  }
+#else
+  tcp->inq_capable = false;
+#endif /* GRPC_HAVE_TCP_INQ */
   /* Start being notified on errors if event engine can track errors. */
   if (grpc_event_engine_can_track_errors()) {
     /* Grab a ref to tcp so that we can safely access the tcp struct when
@@ -1159,7 +1282,6 @@ int grpc_tcp_fd(grpc_endpoint* ep) {
 
 void grpc_tcp_destroy_and_release_fd(grpc_endpoint* ep, int* fd,
                                      grpc_closure* done) {
-  grpc_network_status_unregister_endpoint(ep);
   grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep);
   GPR_ASSERT(ep->vtable == &vtable);
   tcp->release_fd = fd;
@@ -1167,12 +1289,6 @@ void grpc_tcp_destroy_and_release_fd(grpc_endpoint* ep, int* fd,
   grpc_slice_buffer_reset_and_unref_internal(&tcp->last_read_buffer);
   if (grpc_event_engine_can_track_errors()) {
     /* Stop errors notification. */
-    gpr_mu_lock(&tcp->tb_mu);
-    grpc_core::TracedBuffer::Shutdown(
-        &tcp->tb_head, tcp->outgoing_buffer_arg,
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("endpoint destroyed"));
-    gpr_mu_unlock(&tcp->tb_mu);
-    tcp->outgoing_buffer_arg = nullptr;
     gpr_atm_no_barrier_store(&tcp->stop_error_notification, true);
     grpc_fd_set_error(tcp->em_fd);
   }