grpc 1.18.0 → 1.19.0

data/src/core/lib/http/httpcli_security_connector.cc

@@ -67,7 +67,7 @@ class grpc_httpcli_ssl_channel_security_connector final
   }
 
   void add_handshakers(grpc_pollset_set* interested_parties,
-                       grpc_handshake_manager* handshake_mgr) override {
+                       grpc_core::HandshakeManager* handshake_mgr) override {
     tsi_handshaker* handshaker = nullptr;
     if (handshaker_factory_ != nullptr) {
       tsi_result result = tsi_ssl_client_handshaker_factory_create_handshaker(
@@ -77,8 +77,7 @@ class grpc_httpcli_ssl_channel_security_connector final
                 tsi_result_to_string(result));
       }
     }
-    grpc_handshake_manager_add(
-        handshake_mgr, grpc_security_handshaker_create(handshaker, this));
+    handshake_mgr->Add(grpc_core::SecurityHandshakerCreate(handshaker, this));
   }
 
   tsi_ssl_client_handshaker_factory* handshaker_factory() const {
@@ -155,11 +154,11 @@ httpcli_ssl_channel_security_connector_create(
 typedef struct {
   void (*func)(void* arg, grpc_endpoint* endpoint);
   void* arg;
-  grpc_handshake_manager* handshake_mgr;
+  grpc_core::RefCountedPtr<grpc_core::HandshakeManager> handshake_mgr;
 } on_done_closure;
 
 static void on_handshake_done(void* arg, grpc_error* error) {
-  grpc_handshaker_args* args = static_cast<grpc_handshaker_args*>(arg);
+  auto* args = static_cast<grpc_core::HandshakerArgs*>(arg);
   on_done_closure* c = static_cast<on_done_closure*>(args->user_data);
   if (error != GRPC_ERROR_NONE) {
     const char* msg = grpc_error_string(error);
@@ -172,14 +171,13 @@ static void on_handshake_done(void* arg, grpc_error* error) {
     gpr_free(args->read_buffer);
     c->func(c->arg, args->endpoint);
   }
-  grpc_handshake_manager_destroy(c->handshake_mgr);
-  gpr_free(c);
+  grpc_core::Delete<on_done_closure>(c);
 }
 
 static void ssl_handshake(void* arg, grpc_endpoint* tcp, const char* host,
                           grpc_millis deadline,
                           void (*on_done)(void* arg, grpc_endpoint* endpoint)) {
-  on_done_closure* c = static_cast<on_done_closure*>(gpr_malloc(sizeof(*c)));
+  auto* c = grpc_core::New<on_done_closure>();
   const char* pem_root_certs =
       grpc_core::DefaultSslRootStore::GetPemRootCerts();
   const tsi_ssl_root_certs_store* root_store =
@@ -198,12 +196,13 @@ static void ssl_handshake(void* arg, grpc_endpoint* tcp, const char* host,
   GPR_ASSERT(sc != nullptr);
   grpc_arg channel_arg = grpc_security_connector_to_arg(sc.get());
   grpc_channel_args args = {1, &channel_arg};
-  c->handshake_mgr = grpc_handshake_manager_create();
-  grpc_handshakers_add(HANDSHAKER_CLIENT, &args,
-                       nullptr /* interested_parties */, c->handshake_mgr);
-  grpc_handshake_manager_do_handshake(
-      c->handshake_mgr, tcp, nullptr /* channel_args */, deadline,
-      nullptr /* acceptor */, on_handshake_done, c /* user_data */);
+  c->handshake_mgr = grpc_core::MakeRefCounted<grpc_core::HandshakeManager>();
+  grpc_core::HandshakerRegistry::AddHandshakers(
+      grpc_core::HANDSHAKER_CLIENT, &args, /*interested_parties=*/nullptr,
+      c->handshake_mgr.get());
+  c->handshake_mgr->DoHandshake(tcp, /*channel_args=*/nullptr, deadline,
+                                /*acceptor=*/nullptr, on_handshake_done,
+                                /*user_data=*/c);
   sc.reset(DEBUG_LOCATION, "httpcli");
 }
 

data/src/core/lib/iomgr/buffer_list.cc

@@ -24,32 +24,13 @@
 #include <grpc/support/log.h>
 
 #ifdef GRPC_LINUX_ERRQUEUE
+#include <netinet/in.h>
+#include <string.h>
 #include <time.h>
 
 #include "src/core/lib/gprpp/memory.h"
 
 namespace grpc_core {
-void TracedBuffer::AddNewEntry(TracedBuffer** head, uint32_t seq_no,
-                               void* arg) {
-  GPR_DEBUG_ASSERT(head != nullptr);
-  TracedBuffer* new_elem = New<TracedBuffer>(seq_no, arg);
-  /* Store the current time as the sendmsg time. */
-  new_elem->ts_.sendmsg_time = gpr_now(GPR_CLOCK_REALTIME);
-  new_elem->ts_.scheduled_time = gpr_inf_past(GPR_CLOCK_REALTIME);
-  new_elem->ts_.sent_time = gpr_inf_past(GPR_CLOCK_REALTIME);
-  new_elem->ts_.acked_time = gpr_inf_past(GPR_CLOCK_REALTIME);
-  if (*head == nullptr) {
-    *head = new_elem;
-    return;
-  }
-  /* Append at the end. */
-  TracedBuffer* ptr = *head;
-  while (ptr->next_ != nullptr) {
-    ptr = ptr->next_;
-  }
-  ptr->next_ = new_elem;
-}
-
 namespace {
 /** Fills gpr_timespec gts based on values from timespec ts */
 void fill_gpr_from_timestamp(gpr_timespec* gts, const struct timespec* ts) {
@@ -68,10 +49,180 @@ void default_timestamps_callback(void* arg, grpc_core::Timestamps* ts,
 void (*timestamps_callback)(void*, grpc_core::Timestamps*,
                             grpc_error* shutdown_err) =
     default_timestamps_callback;
+
+/* Used to extract individual opt stats from cmsg, so as to avoid troubles with
+ * unaligned reads */
+template <typename T>
+T read_unaligned(const void* ptr) {
+  T val;
+  memcpy(&val, ptr, sizeof(val));
+  return val;
+}
+
+/* Extracts opt stats from the tcp_info struct \a info to \a metrics */
+void extract_opt_stats_from_tcp_info(ConnectionMetrics* metrics,
+                                     const grpc_core::tcp_info* info) {
+  if (info == nullptr) {
+    return;
+  }
+  if (info->length > offsetof(grpc_core::tcp_info, tcpi_sndbuf_limited)) {
+    metrics->recurring_retrans.set(info->tcpi_retransmits);
+    metrics->is_delivery_rate_app_limited.set(
+        info->tcpi_delivery_rate_app_limited);
+    metrics->congestion_window.set(info->tcpi_snd_cwnd);
+    metrics->reordering.set(info->tcpi_reordering);
+    metrics->packet_retx.set(info->tcpi_total_retrans);
+    metrics->pacing_rate.set(info->tcpi_pacing_rate);
+    metrics->data_notsent.set(info->tcpi_notsent_bytes);
+    if (info->tcpi_min_rtt != UINT32_MAX) {
+      metrics->min_rtt.set(info->tcpi_min_rtt);
+    }
+    metrics->packet_sent.set(info->tcpi_data_segs_out);
+    metrics->delivery_rate.set(info->tcpi_delivery_rate);
+    metrics->busy_usec.set(info->tcpi_busy_time);
+    metrics->rwnd_limited_usec.set(info->tcpi_rwnd_limited);
+    metrics->sndbuf_limited_usec.set(info->tcpi_sndbuf_limited);
+  }
+  if (info->length > offsetof(grpc_core::tcp_info, tcpi_dsack_dups)) {
+    metrics->data_sent.set(info->tcpi_bytes_sent);
+    metrics->data_retx.set(info->tcpi_bytes_retrans);
+    metrics->packet_spurious_retx.set(info->tcpi_dsack_dups);
+  }
+}
+
+/** Extracts opt stats from the given control message \a opt_stats to the
+ * connection metrics \a metrics */
+void extract_opt_stats_from_cmsg(ConnectionMetrics* metrics,
+                                 const cmsghdr* opt_stats) {
+  if (opt_stats == nullptr) {
+    return;
+  }
+  const auto* data = CMSG_DATA(opt_stats);
+  constexpr int64_t cmsg_hdr_len = CMSG_ALIGN(sizeof(struct cmsghdr));
+  const int64_t len = opt_stats->cmsg_len - cmsg_hdr_len;
+  int64_t offset = 0;
+
+  while (offset < len) {
+    const auto* attr = reinterpret_cast<const nlattr*>(data + offset);
+    const void* val = data + offset + NLA_HDRLEN;
+    switch (attr->nla_type) {
+      case TCP_NLA_BUSY: {
+        metrics->busy_usec.set(read_unaligned<uint64_t>(val));
+        break;
+      }
+      case TCP_NLA_RWND_LIMITED: {
+        metrics->rwnd_limited_usec.set(read_unaligned<uint64_t>(val));
+        break;
+      }
+      case TCP_NLA_SNDBUF_LIMITED: {
+        metrics->sndbuf_limited_usec.set(read_unaligned<uint64_t>(val));
+        break;
+      }
+      case TCP_NLA_PACING_RATE: {
+        metrics->pacing_rate.set(read_unaligned<uint64_t>(val));
+        break;
+      }
+      case TCP_NLA_DELIVERY_RATE: {
+        metrics->delivery_rate.set(read_unaligned<uint64_t>(val));
+        break;
+      }
+      case TCP_NLA_DELIVERY_RATE_APP_LMT: {
+        metrics->is_delivery_rate_app_limited.set(read_unaligned<uint8_t>(val));
+        break;
+      }
+      case TCP_NLA_SND_CWND: {
+        metrics->congestion_window.set(read_unaligned<uint32_t>(val));
+        break;
+      }
+      case TCP_NLA_MIN_RTT: {
+        metrics->min_rtt.set(read_unaligned<uint32_t>(val));
+        break;
+      }
+      case TCP_NLA_SRTT: {
+        metrics->srtt.set(read_unaligned<uint32_t>(val));
+        break;
+      }
+      case TCP_NLA_RECUR_RETRANS: {
+        metrics->recurring_retrans.set(read_unaligned<uint8_t>(val));
+        break;
+      }
+      case TCP_NLA_BYTES_SENT: {
+        metrics->data_sent.set(read_unaligned<uint64_t>(val));
+        break;
+      }
+      case TCP_NLA_DATA_SEGS_OUT: {
+        metrics->packet_sent.set(read_unaligned<uint64_t>(val));
+        break;
+      }
+      case TCP_NLA_TOTAL_RETRANS: {
+        metrics->packet_retx.set(read_unaligned<uint64_t>(val));
+        break;
+      }
+      case TCP_NLA_DELIVERED: {
+        metrics->packet_delivered.set(read_unaligned<uint32_t>(val));
+        break;
+      }
+      case TCP_NLA_DELIVERED_CE: {
+        metrics->packet_delivered_ce.set(read_unaligned<uint32_t>(val));
+        break;
+      }
+      case TCP_NLA_BYTES_RETRANS: {
+        metrics->data_retx.set(read_unaligned<uint64_t>(val));
+        break;
+      }
+      case TCP_NLA_DSACK_DUPS: {
+        metrics->packet_spurious_retx.set(read_unaligned<uint32_t>(val));
+        break;
+      }
+      case TCP_NLA_REORDERING: {
+        metrics->reordering.set(read_unaligned<uint32_t>(val));
+        break;
+      }
+      case TCP_NLA_SND_SSTHRESH: {
+        metrics->snd_ssthresh.set(read_unaligned<uint32_t>(val));
+        break;
+      }
+    }
+    offset += NLA_ALIGN(attr->nla_len);
+  }
+}
+
+static int get_socket_tcp_info(grpc_core::tcp_info* info, int fd) {
+  memset(info, 0, sizeof(*info));
+  info->length = sizeof(*info) - sizeof(socklen_t);
+  return getsockopt(fd, IPPROTO_TCP, TCP_INFO, info, &(info->length));
+}
 } /* namespace */
 
+void TracedBuffer::AddNewEntry(TracedBuffer** head, uint32_t seq_no, int fd,
+                               void* arg) {
+  GPR_DEBUG_ASSERT(head != nullptr);
+  TracedBuffer* new_elem = New<TracedBuffer>(seq_no, arg);
+  /* Store the current time as the sendmsg time. */
+  new_elem->ts_.sendmsg_time.time = gpr_now(GPR_CLOCK_REALTIME);
+  new_elem->ts_.scheduled_time.time = gpr_inf_past(GPR_CLOCK_REALTIME);
+  new_elem->ts_.sent_time.time = gpr_inf_past(GPR_CLOCK_REALTIME);
+  new_elem->ts_.acked_time.time = gpr_inf_past(GPR_CLOCK_REALTIME);
+
+  if (get_socket_tcp_info(&new_elem->ts_.info, fd) == 0) {
+    extract_opt_stats_from_tcp_info(&new_elem->ts_.sendmsg_time.metrics,
+                                    &new_elem->ts_.info);
+  }
+  if (*head == nullptr) {
+    *head = new_elem;
+    return;
+  }
+  /* Append at the end. */
+  TracedBuffer* ptr = *head;
+  while (ptr->next_ != nullptr) {
+    ptr = ptr->next_;
+  }
+  ptr->next_ = new_elem;
+}
+
 void TracedBuffer::ProcessTimestamp(TracedBuffer** head,
                                     struct sock_extended_err* serr,
+                                    struct cmsghdr* opt_stats,
                                     struct scm_timestamping* tss) {
   GPR_DEBUG_ASSERT(head != nullptr);
   TracedBuffer* elem = *head;
@@ -82,15 +233,22 @@ void TracedBuffer::ProcessTimestamp(TracedBuffer** head,
     if (serr->ee_data >= elem->seq_no_) {
       switch (serr->ee_info) {
         case SCM_TSTAMP_SCHED:
-          fill_gpr_from_timestamp(&(elem->ts_.scheduled_time), &(tss->ts[0]));
+          fill_gpr_from_timestamp(&(elem->ts_.scheduled_time.time),
+                                  &(tss->ts[0]));
+          extract_opt_stats_from_cmsg(&(elem->ts_.scheduled_time.metrics),
+                                      opt_stats);
           elem = elem->next_;
           break;
         case SCM_TSTAMP_SND:
-          fill_gpr_from_timestamp(&(elem->ts_.sent_time), &(tss->ts[0]));
+          fill_gpr_from_timestamp(&(elem->ts_.sent_time.time), &(tss->ts[0]));
+          extract_opt_stats_from_cmsg(&(elem->ts_.sent_time.metrics),
+                                      opt_stats);
           elem = elem->next_;
           break;
         case SCM_TSTAMP_ACK:
-          fill_gpr_from_timestamp(&(elem->ts_.acked_time), &(tss->ts[0]));
+          fill_gpr_from_timestamp(&(elem->ts_.acked_time.time), &(tss->ts[0]));
+          extract_opt_stats_from_cmsg(&(elem->ts_.acked_time.metrics),
+                                      opt_stats);
           /* Got all timestamps. Do the callback and free this TracedBuffer.
            * The thing below can be passed by value if we don't want the
            * restriction on the lifetime. */

data/src/core/lib/iomgr/buffer_list.h

@@ -26,19 +26,78 @@
 #include <grpc/support/time.h>
 
 #include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/gprpp/optional.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/internal_errqueue.h"
 
 namespace grpc_core {
+
+struct ConnectionMetrics {
+  /* Delivery rate in Bytes/s. */
+  Optional<uint64_t> delivery_rate;
+  /* If the delivery rate is limited by the application, this is set to true. */
+  Optional<bool> is_delivery_rate_app_limited;
+  /* Total packets retransmitted. */
+  Optional<uint32_t> packet_retx;
+  /* Total packets retransmitted spuriously. This metric is smaller than or
+  equal to packet_retx. */
+  Optional<uint32_t> packet_spurious_retx;
+  /* Total packets sent. */
+  Optional<uint32_t> packet_sent;
+  /* Total packets delivered. */
+  Optional<uint32_t> packet_delivered;
+  /* Total packets delivered with ECE marked. This metric is smaller than or
+  equal to packet_delivered. */
+  Optional<uint32_t> packet_delivered_ce;
+  /* Total bytes lost so far. */
+  Optional<uint64_t> data_retx;
+  /* Total bytes sent so far. */
+  Optional<uint64_t> data_sent;
+  /* Total bytes in write queue but not sent. */
+  Optional<uint64_t> data_notsent;
+  /* Pacing rate of the connection in Bps */
+  Optional<uint64_t> pacing_rate;
+  /* Minimum RTT observed in usec. */
+  Optional<uint32_t> min_rtt;
+  /* Smoothed RTT in usec */
+  Optional<uint32_t> srtt;
+  /* Send congestion window. */
+  Optional<uint32_t> congestion_window;
+  /* Slow start threshold in packets. */
+  Optional<uint32_t> snd_ssthresh;
+  /* Maximum degree of reordering (i.e., maximum number of packets reodered)
+   on the connection. */
+  Optional<uint32_t> reordering;
+  /* Represents the number of recurring retransmissions of the first sequence
+  that is not acknowledged yet. */
+  Optional<uint8_t> recurring_retrans;
+  /* The cumulative time (in usec) that the transport protocol was busy
+   sending data. */
+  Optional<uint64_t> busy_usec;
+  /* The cumulative time (in usec) that the transport protocol was limited by
+   the receive window size. */
+  Optional<uint64_t> rwnd_limited_usec;
+  /* The cumulative time (in usec) that the transport protocol was limited by
+   the send buffer size. */
+  Optional<uint64_t> sndbuf_limited_usec;
+};
+
+struct Timestamp {
+  gpr_timespec time;
+  ConnectionMetrics metrics; /* Metrics collected with this timestamp */
+};
+
 struct Timestamps {
-  /* TODO(yashykt): This would also need to store OPTSTAT once support is added
-   */
-  gpr_timespec sendmsg_time;
-  gpr_timespec scheduled_time;
-  gpr_timespec sent_time;
-  gpr_timespec acked_time;
+  Timestamp sendmsg_time;
+  Timestamp scheduled_time;
+  Timestamp sent_time;
+  Timestamp acked_time;
 
   uint32_t byte_offset; /* byte offset relative to the start of the RPC */
+
+#ifdef GRPC_LINUX_ERRQUEUE
+  grpc_core::tcp_info info; /* tcp_info collected on sendmsg */
+#endif                      /* GRPC_LINUX_ERRQUEUE */
 };
 
 /** TracedBuffer is a class to keep track of timestamps for a specific buffer in
@@ -58,13 +117,14 @@ class TracedBuffer {
   /** Add a new entry in the TracedBuffer list pointed to by head. Also saves
    * sendmsg_time with the current timestamp. */
   static void AddNewEntry(grpc_core::TracedBuffer** head, uint32_t seq_no,
-                          void* arg);
+                          int fd, void* arg);
 
   /** Processes a received timestamp based on sock_extended_err and
    * scm_timestamping structures. It will invoke the timestamps callback if the
    * timestamp type is SCM_TSTAMP_ACK. */
   static void ProcessTimestamp(grpc_core::TracedBuffer** head,
                                struct sock_extended_err* serr,
+                               struct cmsghdr* opt_stats,
                                struct scm_timestamping* tss);
 
   /** Cleans the list by calling the callback for each traced buffer in the list
@@ -88,7 +148,9 @@ class TracedBuffer {
  public:
   /* Dummy shutdown function */
   static void Shutdown(grpc_core::TracedBuffer** head, void* remaining,
-                       grpc_error* shutdown_err) {}
+                       grpc_error* shutdown_err) {
+    GRPC_ERROR_UNREF(shutdown_err);
+  }
 };
 #endif /* GRPC_LINUX_ERRQUEUE */
 

data/src/core/lib/iomgr/combiner.cc

@@ -29,6 +29,7 @@
 
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/profiling/timers.h"
 
 grpc_core::DebugOnlyTraceFlag grpc_combiner_trace(false, "combiner");
@@ -82,8 +83,9 @@ grpc_combiner* grpc_combiner_create(void) {
   gpr_atm_no_barrier_store(&lock->state, STATE_UNORPHANED);
   gpr_mpscq_init(&lock->queue);
   grpc_closure_list_init(&lock->final_list);
-  GRPC_CLOSURE_INIT(&lock->offload, offload, lock,
-                    grpc_executor_scheduler(GRPC_EXECUTOR_SHORT));
+  GRPC_CLOSURE_INIT(
+      &lock->offload, offload, lock,
+      grpc_core::Executor::Scheduler(grpc_core::ExecutorJobType::SHORT));
   GRPC_COMBINER_TRACE(gpr_log(GPR_INFO, "C:%p create", lock));
   return lock;
 }
@@ -228,8 +230,14 @@ bool grpc_combiner_continue_exec_ctx() {
                               grpc_core::ExecCtx::Get()->IsReadyToFinish(),
                               lock->time_to_execute_final_list));
 
+  // offload only if all the following conditions are true:
+  // 1. the combiner is contended and has more than one closure to execute
+  // 2. the current execution context needs to finish as soon as possible
+  // 3. the DEFAULT executor is threaded
+  // 4. the current thread is not a worker for any background poller
   if (contended && grpc_core::ExecCtx::Get()->IsReadyToFinish() &&
-      grpc_executor_is_threaded()) {
+      grpc_core::Executor::IsThreadedDefault() &&
+      !grpc_iomgr_is_any_background_poller_thread()) {
     GPR_TIMER_MARK("offload_from_finished_exec_ctx", 0);
     // this execution context wants to move on: schedule remaining work to be
     // picked up on the executor

data/src/core/lib/iomgr/error.cc

@@ -303,11 +303,15 @@ static void internal_add_error(grpc_error** err, grpc_error* new_err) {
 // It is very common to include and extra int and string in an error
 #define SURPLUS_CAPACITY (2 * SLOTS_PER_INT + SLOTS_PER_TIME)
 
-static bool g_error_creation_allowed = true;
+static gpr_atm g_error_creation_allowed = true;
 
-void grpc_disable_error_creation() { g_error_creation_allowed = false; }
+void grpc_disable_error_creation() {
+  gpr_atm_no_barrier_store(&g_error_creation_allowed, false);
+}
 
-void grpc_enable_error_creation() { g_error_creation_allowed = true; }
+void grpc_enable_error_creation() {
+  gpr_atm_no_barrier_store(&g_error_creation_allowed, true);
+}
 
 grpc_error* grpc_error_create(const char* file, int line, grpc_slice desc,
                               grpc_error** referencing,
@@ -323,7 +327,7 @@ grpc_error* grpc_error_create(const char* file, int line, grpc_slice desc,
     return GRPC_ERROR_OOM;
   }
 #ifndef NDEBUG
-  if (!g_error_creation_allowed) {
+  if (!gpr_atm_no_barrier_load(&g_error_creation_allowed)) {
     gpr_log(GPR_ERROR,
             "Error creation occurred when error creation was disabled [%s:%d]",
             file, line);
@@ -765,7 +769,7 @@ grpc_error* grpc_os_error(const char* file, int line, int err,
       grpc_error_set_str(
           grpc_error_set_int(
               grpc_error_create(file, line,
-                                grpc_slice_from_static_string("OS Error"),
+                                grpc_slice_from_static_string(strerror(err)),
                                 nullptr, 0),
               GRPC_ERROR_INT_ERRNO, err),
           GRPC_ERROR_STR_OS_ERROR,

data/src/core/lib/iomgr/ev_epoll1_linux.cc

@@ -1242,6 +1242,8 @@ static void pollset_set_del_pollset_set(grpc_pollset_set* bag,
  * Event engine binding
  */
 
+static bool is_any_background_poller_thread(void) { return false; }
+
 static void shutdown_background_closure(void) {}
 
 static void shutdown_engine(void) {
@@ -1287,6 +1289,7 @@ static const grpc_event_engine_vtable vtable = {
     pollset_set_add_fd,
     pollset_set_del_fd,
 
+    is_any_background_poller_thread,
     shutdown_background_closure,
     shutdown_engine,
 };