grpc 1.26.0 → 1.28.0

data/src/core/lib/gprpp/sync.h

@@ -117,6 +117,15 @@ class CondVar {
     }
   }
 
+  // Returns true iff we timed-out
+  template <typename Predicate>
+  bool WaitUntil(Mutex* mu, Predicate pred, const gpr_timespec& deadline) {
+    while (!pred()) {
+      if (Wait(mu, deadline)) return true;
+    }
+    return false;
+  }
+
  private:
   gpr_cv cv_;
 };

data/src/core/lib/iomgr/buffer_list.cc

@@ -66,27 +66,27 @@ void extract_opt_stats_from_tcp_info(ConnectionMetrics* metrics,
     return;
   }
   if (info->length > offsetof(grpc_core::tcp_info, tcpi_sndbuf_limited)) {
-    metrics->recurring_retrans.set(info->tcpi_retransmits);
-    metrics->is_delivery_rate_app_limited.set(
+    metrics->recurring_retrans.emplace(info->tcpi_retransmits);
+    metrics->is_delivery_rate_app_limited.emplace(
         info->tcpi_delivery_rate_app_limited);
-    metrics->congestion_window.set(info->tcpi_snd_cwnd);
-    metrics->reordering.set(info->tcpi_reordering);
-    metrics->packet_retx.set(info->tcpi_total_retrans);
-    metrics->pacing_rate.set(info->tcpi_pacing_rate);
-    metrics->data_notsent.set(info->tcpi_notsent_bytes);
+    metrics->congestion_window.emplace(info->tcpi_snd_cwnd);
+    metrics->reordering.emplace(info->tcpi_reordering);
+    metrics->packet_retx.emplace(info->tcpi_total_retrans);
+    metrics->pacing_rate.emplace(info->tcpi_pacing_rate);
+    metrics->data_notsent.emplace(info->tcpi_notsent_bytes);
     if (info->tcpi_min_rtt != UINT32_MAX) {
-      metrics->min_rtt.set(info->tcpi_min_rtt);
+      metrics->min_rtt.emplace(info->tcpi_min_rtt);
     }
-    metrics->packet_sent.set(info->tcpi_data_segs_out);
-    metrics->delivery_rate.set(info->tcpi_delivery_rate);
-    metrics->busy_usec.set(info->tcpi_busy_time);
-    metrics->rwnd_limited_usec.set(info->tcpi_rwnd_limited);
-    metrics->sndbuf_limited_usec.set(info->tcpi_sndbuf_limited);
+    metrics->packet_sent.emplace(info->tcpi_data_segs_out);
+    metrics->delivery_rate.emplace(info->tcpi_delivery_rate);
+    metrics->busy_usec.emplace(info->tcpi_busy_time);
+    metrics->rwnd_limited_usec.emplace(info->tcpi_rwnd_limited);
+    metrics->sndbuf_limited_usec.emplace(info->tcpi_sndbuf_limited);
   }
   if (info->length > offsetof(grpc_core::tcp_info, tcpi_dsack_dups)) {
-    metrics->data_sent.set(info->tcpi_bytes_sent);
-    metrics->data_retx.set(info->tcpi_bytes_retrans);
-    metrics->packet_spurious_retx.set(info->tcpi_dsack_dups);
+    metrics->data_sent.emplace(info->tcpi_bytes_sent);
+    metrics->data_retx.emplace(info->tcpi_bytes_retrans);
+    metrics->packet_spurious_retx.emplace(info->tcpi_dsack_dups);
   }
 }
 
@@ -107,79 +107,80 @@ void extract_opt_stats_from_cmsg(ConnectionMetrics* metrics,
     const void* val = data + offset + NLA_HDRLEN;
     switch (attr->nla_type) {
       case TCP_NLA_BUSY: {
-        metrics->busy_usec.set(read_unaligned<uint64_t>(val));
+        metrics->busy_usec.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_RWND_LIMITED: {
-        metrics->rwnd_limited_usec.set(read_unaligned<uint64_t>(val));
+        metrics->rwnd_limited_usec.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_SNDBUF_LIMITED: {
-        metrics->sndbuf_limited_usec.set(read_unaligned<uint64_t>(val));
+        metrics->sndbuf_limited_usec.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_PACING_RATE: {
-        metrics->pacing_rate.set(read_unaligned<uint64_t>(val));
+        metrics->pacing_rate.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_DELIVERY_RATE: {
-        metrics->delivery_rate.set(read_unaligned<uint64_t>(val));
+        metrics->delivery_rate.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_DELIVERY_RATE_APP_LMT: {
-        metrics->is_delivery_rate_app_limited.set(read_unaligned<uint8_t>(val));
+        metrics->is_delivery_rate_app_limited.emplace(
+            read_unaligned<uint8_t>(val));
         break;
       }
       case TCP_NLA_SND_CWND: {
-        metrics->congestion_window.set(read_unaligned<uint32_t>(val));
+        metrics->congestion_window.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_MIN_RTT: {
-        metrics->min_rtt.set(read_unaligned<uint32_t>(val));
+        metrics->min_rtt.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_SRTT: {
-        metrics->srtt.set(read_unaligned<uint32_t>(val));
+        metrics->srtt.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_RECUR_RETRANS: {
-        metrics->recurring_retrans.set(read_unaligned<uint8_t>(val));
+        metrics->recurring_retrans.emplace(read_unaligned<uint8_t>(val));
         break;
       }
       case TCP_NLA_BYTES_SENT: {
-        metrics->data_sent.set(read_unaligned<uint64_t>(val));
+        metrics->data_sent.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_DATA_SEGS_OUT: {
-        metrics->packet_sent.set(read_unaligned<uint64_t>(val));
+        metrics->packet_sent.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_TOTAL_RETRANS: {
-        metrics->packet_retx.set(read_unaligned<uint64_t>(val));
+        metrics->packet_retx.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_DELIVERED: {
-        metrics->packet_delivered.set(read_unaligned<uint32_t>(val));
+        metrics->packet_delivered.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_DELIVERED_CE: {
-        metrics->packet_delivered_ce.set(read_unaligned<uint32_t>(val));
+        metrics->packet_delivered_ce.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_BYTES_RETRANS: {
-        metrics->data_retx.set(read_unaligned<uint64_t>(val));
+        metrics->data_retx.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_DSACK_DUPS: {
-        metrics->packet_spurious_retx.set(read_unaligned<uint32_t>(val));
+        metrics->packet_spurious_retx.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_REORDERING: {
-        metrics->reordering.set(read_unaligned<uint32_t>(val));
+        metrics->reordering.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_SND_SSTHRESH: {
-        metrics->snd_ssthresh.set(read_unaligned<uint32_t>(val));
+        metrics->snd_ssthresh.emplace(read_unaligned<uint32_t>(val));
         break;
       }
     }

data/src/core/lib/iomgr/error.h

@@ -201,10 +201,10 @@ inline void grpc_error_unref(grpc_error* err) {
 
 // Consumes all the errors in the vector and forms a referencing error from
 // them. If the vector is empty, return GRPC_ERROR_NONE.
-template <size_t N>
-static grpc_error* grpc_error_create_from_vector(
-    const char* file, int line, const char* desc,
-    grpc_core::InlinedVector<grpc_error*, N>* error_list) {
+template <typename VectorType>
+static grpc_error* grpc_error_create_from_vector(const char* file, int line,
+                                                 const char* desc,
+                                                 VectorType* error_list) {
   grpc_error* error = GRPC_ERROR_NONE;
   if (error_list->size() != 0) {
     error = grpc_error_create(file, line, grpc_slice_from_static_string(desc),

data/src/core/lib/iomgr/ev_epollex_linux.cc

@@ -41,11 +41,11 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
 
+#include "absl/container/inlined_vector.h"
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/gpr/spinlock.h"
 #include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gpr/useful.h"
-#include "src/core/lib/gprpp/inlined_vector.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/sync.h"
@@ -190,7 +190,15 @@ struct grpc_fd {
     grpc_iomgr_unregister_object(&iomgr_object);
 
     POLLABLE_UNREF(pollable_obj, "fd_pollable");
-    pollset_fds.clear();
+
+    // To clear out the allocations of pollset_fds, we need to swap its
+    // contents with a newly-constructed (and soon to be destructed) local
+    // variable of its same type. This is because InlinedVector::clear is _not_
+    // guaranteed to actually free up allocations and this is important since
+    // this object doesn't have a conventional destructor.
+    absl::InlinedVector<int, 1> pollset_fds_tmp;
+    pollset_fds_tmp.swap(pollset_fds);
+
     gpr_mu_destroy(&pollable_mu);
     gpr_mu_destroy(&orphan_mu);
 
@@ -232,8 +240,8 @@ struct grpc_fd {
 
   // Protects pollable_obj and pollset_fds.
   gpr_mu pollable_mu;
-  grpc_core::InlinedVector<int, 1> pollset_fds;  // Used in PO_MULTI.
-  pollable* pollable_obj = nullptr;              // Used in PO_FD.
+  absl::InlinedVector<int, 1> pollset_fds;  // Used in PO_MULTI.
+  pollable* pollable_obj = nullptr;         // Used in PO_FD.
 
   grpc_core::LockfreeEvent read_closure;
   grpc_core::LockfreeEvent write_closure;

data/src/core/lib/iomgr/executor.cc

@@ -143,7 +143,7 @@ void Executor::SetThreading(bool threading) {
 
   if (threading) {
     if (curr_num_threads > 0) {
-      EXECUTOR_TRACE("(%s) SetThreading(true). curr_num_threads == 0", name_);
+      EXECUTOR_TRACE("(%s) SetThreading(true). curr_num_threads > 0", name_);
       return;
     }
 

data/src/core/lib/iomgr/fork_posix.cc

@@ -22,6 +22,10 @@
 
 #ifdef GRPC_POSIX_FORK
 
+#ifdef GRPC_POSIX_FORK_ALLOW_PTHREAD_ATFORK
+#include <pthread.h>
+#endif
+
 #include <string.h>
 
 #include <grpc/fork.h>

data/src/core/lib/iomgr/load_file.cc

@@ -53,6 +53,7 @@ grpc_error* grpc_load_file(const char* filename, int add_null_terminator,
       gpr_malloc(contents_size + (add_null_terminator ? 1 : 0)));
   bytes_read = fread(contents, 1, contents_size, file);
   if (bytes_read < contents_size) {
+    gpr_free(contents);
     error = GRPC_OS_ERROR(errno, "fread");
     GPR_ASSERT(ferror(file));
     goto end;

data/src/core/lib/iomgr/poller/eventmanager_libuv.cc

@@ -0,0 +1,87 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/iomgr/poller/eventmanager_libuv.h"
+
+#include <grpc/support/time.h>
+
+grpc::experimental::LibuvEventManager::Options::Options() : num_workers_(-1) {}
+grpc::experimental::LibuvEventManager::Options::Options(int num_workers)
+    : num_workers_(num_workers) {}
+
+grpc::experimental::LibuvEventManager::LibuvEventManager(const Options& options)
+    : options_(options) {
+  int num_workers = options_.num_workers();
+  // Number of workers can't be 0 if we do not accept thread donation.
+  // TODO(guantaol): replaces the hard-coded number with a flag.
+  if (num_workers <= 0) num_workers = 32;
+
+  for (int i = 0; i < num_workers; i++) {
+    workers_.emplace_back(
+        options_.thread_name_prefix().c_str(),
+        [](void* em) { static_cast<LibuvEventManager*>(em)->RunWorkerLoop(); },
+        this);
+    workers_.back().Start();
+  }
+}
+
+grpc::experimental::LibuvEventManager::~LibuvEventManager() {
+  Shutdown();
+  for (auto& th : workers_) {
+    th.Join();
+  }
+}
+
+void grpc::experimental::LibuvEventManager::RunWorkerLoop() {
+  while (true) {
+    // TODO(guantaol): extend the worker loop with real work.
+    if (ShouldStop()) return;
+    gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
+                                 gpr_time_from_micros(10, GPR_TIMESPAN)));
+  }
+}
+
+bool grpc::experimental::LibuvEventManager::ShouldStop() {
+  return should_stop_.Load(grpc_core::MemoryOrder::ACQUIRE) != 0;
+}
+
+void grpc::experimental::LibuvEventManager::Shutdown() {
+  if (should_stop_.Load(grpc_core::MemoryOrder::ACQUIRE))
+    return;  // Already shut down.
+
+  {
+    grpc_core::MutexLock lock(&shutdown_mu_);
+    while (shutdown_refcount_.Load(grpc_core::MemoryOrder::ACQUIRE) > 0) {
+      shutdown_cv_.Wait(&shutdown_mu_);
+    }
+  }
+  should_stop_.Store(true, grpc_core::MemoryOrder::RELEASE);
+}
+
+void grpc::experimental::LibuvEventManager::ShutdownRef() {
+  shutdown_refcount_.FetchAdd(1, grpc_core::MemoryOrder::RELAXED);
+}
+
+void grpc::experimental::LibuvEventManager::ShutdownUnref() {
+  if (shutdown_refcount_.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1) {
+    grpc_core::MutexLock lock(&shutdown_mu_);
+    shutdown_cv_.Signal();
+  }
+}

data/src/core/lib/iomgr/poller/eventmanager_libuv.h

@@ -0,0 +1,88 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_POLLER_EVENTMANAGER_LIBUV_H
+#define GRPC_CORE_LIB_IOMGR_POLLER_EVENTMANAGER_LIBUV_H
+
+#include <grpc/support/port_platform.h>
+
+#include <string>
+#include <vector>
+
+#include "src/core/lib/gprpp/atomic.h"
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/gprpp/thd.h"
+
+namespace grpc {
+namespace experimental {
+
+class LibuvEventManager {
+ public:
+  class Options {
+   public:
+    Options();
+    Options(int num_workers);
+
+    int num_workers() const { return num_workers_; }
+    void set_num_workers(int num) { num_workers_ = num; }
+
+    const std::string& thread_name_prefix() const {
+      return thread_name_prefix_;
+    }
+    void set_thread_name_prefix(const std::string& name) {
+      thread_name_prefix_ = name;
+    }
+
+   private:
+    // Number of worker threads to create at startup. If less than 0, uses the
+    // default value of 32.
+    int num_workers_;
+    // Name prefix used for worker.
+    std::string thread_name_prefix_;
+  };
+
+  explicit LibuvEventManager(const Options& options);
+  virtual ~LibuvEventManager();
+
+  void Shutdown();
+  void ShutdownRef();
+  void ShutdownUnref();
+
+ private:
+  // Function run by the worker threads.
+  void RunWorkerLoop();
+
+  // Whether the EventManager has been shut down.
+  bool ShouldStop();
+
+  const Options options_;
+  // Whether the EventManager workers should be stopped.
+  grpc_core::Atomic<bool> should_stop_{false};
+  // A refcount preventing the EventManager from shutdown.
+  grpc_core::Atomic<int> shutdown_refcount_{0};
+  // Worker threads of the EventManager.
+  std::vector<grpc_core::Thread> workers_;
+  // Mutex and condition variable used for shutdown.
+  grpc_core::Mutex shutdown_mu_;
+  grpc_core::CondVar shutdown_cv_;
+};
+
+}  // namespace experimental
+}  // namespace grpc
+
+#endif /* GRPC_CORE_LIB_IOMGR_POLLER_EVENTMANAGER_LIBUV_H */

data/src/core/lib/iomgr/socket_utils_common_posix.cc

@@ -50,6 +50,20 @@
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 
+/* set a socket to use zerocopy */
+grpc_error* grpc_set_socket_zerocopy(int fd) {
+#ifdef GRPC_LINUX_ERRQUEUE
+  const int enable = 1;
+  auto err = setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, &enable, sizeof(enable));
+  if (err != 0) {
+    return GRPC_OS_ERROR(errno, "setsockopt(SO_ZEROCOPY)");
+  }
+  return GRPC_ERROR_NONE;
+#else
+  return GRPC_OS_ERROR(ENOSYS, "setsockopt(SO_ZEROCOPY)");
+#endif
+}
+
 /* set a socket to non blocking mode */
 grpc_error* grpc_set_socket_nonblocking(int fd, int non_blocking) {
   int oldflags = fcntl(fd, F_GETFL, 0);