grpc 1.27.0 → 1.28.0

data/src/core/lib/gprpp/sync.h

@@ -117,6 +117,15 @@ class CondVar {
     }
   }
 
+  // Returns true iff we timed-out
+  template <typename Predicate>
+  bool WaitUntil(Mutex* mu, Predicate pred, const gpr_timespec& deadline) {
+    while (!pred()) {
+      if (Wait(mu, deadline)) return true;
+    }
+    return false;
+  }
+
  private:
   gpr_cv cv_;
 };

data/src/core/lib/iomgr/buffer_list.cc

@@ -66,27 +66,27 @@ void extract_opt_stats_from_tcp_info(ConnectionMetrics* metrics,
     return;
   }
   if (info->length > offsetof(grpc_core::tcp_info, tcpi_sndbuf_limited)) {
-    metrics->recurring_retrans.set(info->tcpi_retransmits);
-    metrics->is_delivery_rate_app_limited.set(
+    metrics->recurring_retrans.emplace(info->tcpi_retransmits);
+    metrics->is_delivery_rate_app_limited.emplace(
         info->tcpi_delivery_rate_app_limited);
-    metrics->congestion_window.set(info->tcpi_snd_cwnd);
-    metrics->reordering.set(info->tcpi_reordering);
-    metrics->packet_retx.set(info->tcpi_total_retrans);
-    metrics->pacing_rate.set(info->tcpi_pacing_rate);
-    metrics->data_notsent.set(info->tcpi_notsent_bytes);
+    metrics->congestion_window.emplace(info->tcpi_snd_cwnd);
+    metrics->reordering.emplace(info->tcpi_reordering);
+    metrics->packet_retx.emplace(info->tcpi_total_retrans);
+    metrics->pacing_rate.emplace(info->tcpi_pacing_rate);
+    metrics->data_notsent.emplace(info->tcpi_notsent_bytes);
     if (info->tcpi_min_rtt != UINT32_MAX) {
-      metrics->min_rtt.set(info->tcpi_min_rtt);
+      metrics->min_rtt.emplace(info->tcpi_min_rtt);
     }
-    metrics->packet_sent.set(info->tcpi_data_segs_out);
-    metrics->delivery_rate.set(info->tcpi_delivery_rate);
-    metrics->busy_usec.set(info->tcpi_busy_time);
-    metrics->rwnd_limited_usec.set(info->tcpi_rwnd_limited);
-    metrics->sndbuf_limited_usec.set(info->tcpi_sndbuf_limited);
+    metrics->packet_sent.emplace(info->tcpi_data_segs_out);
+    metrics->delivery_rate.emplace(info->tcpi_delivery_rate);
+    metrics->busy_usec.emplace(info->tcpi_busy_time);
+    metrics->rwnd_limited_usec.emplace(info->tcpi_rwnd_limited);
+    metrics->sndbuf_limited_usec.emplace(info->tcpi_sndbuf_limited);
   }
   if (info->length > offsetof(grpc_core::tcp_info, tcpi_dsack_dups)) {
-    metrics->data_sent.set(info->tcpi_bytes_sent);
-    metrics->data_retx.set(info->tcpi_bytes_retrans);
-    metrics->packet_spurious_retx.set(info->tcpi_dsack_dups);
+    metrics->data_sent.emplace(info->tcpi_bytes_sent);
+    metrics->data_retx.emplace(info->tcpi_bytes_retrans);
+    metrics->packet_spurious_retx.emplace(info->tcpi_dsack_dups);
   }
 }
 
@@ -107,79 +107,80 @@ void extract_opt_stats_from_cmsg(ConnectionMetrics* metrics,
     const void* val = data + offset + NLA_HDRLEN;
     switch (attr->nla_type) {
       case TCP_NLA_BUSY: {
-        metrics->busy_usec.set(read_unaligned<uint64_t>(val));
+        metrics->busy_usec.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_RWND_LIMITED: {
-        metrics->rwnd_limited_usec.set(read_unaligned<uint64_t>(val));
+        metrics->rwnd_limited_usec.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_SNDBUF_LIMITED: {
-        metrics->sndbuf_limited_usec.set(read_unaligned<uint64_t>(val));
+        metrics->sndbuf_limited_usec.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_PACING_RATE: {
-        metrics->pacing_rate.set(read_unaligned<uint64_t>(val));
+        metrics->pacing_rate.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_DELIVERY_RATE: {
-        metrics->delivery_rate.set(read_unaligned<uint64_t>(val));
+        metrics->delivery_rate.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_DELIVERY_RATE_APP_LMT: {
-        metrics->is_delivery_rate_app_limited.set(read_unaligned<uint8_t>(val));
+        metrics->is_delivery_rate_app_limited.emplace(
+            read_unaligned<uint8_t>(val));
         break;
       }
       case TCP_NLA_SND_CWND: {
-        metrics->congestion_window.set(read_unaligned<uint32_t>(val));
+        metrics->congestion_window.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_MIN_RTT: {
-        metrics->min_rtt.set(read_unaligned<uint32_t>(val));
+        metrics->min_rtt.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_SRTT: {
-        metrics->srtt.set(read_unaligned<uint32_t>(val));
+        metrics->srtt.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_RECUR_RETRANS: {
-        metrics->recurring_retrans.set(read_unaligned<uint8_t>(val));
+        metrics->recurring_retrans.emplace(read_unaligned<uint8_t>(val));
         break;
       }
       case TCP_NLA_BYTES_SENT: {
-        metrics->data_sent.set(read_unaligned<uint64_t>(val));
+        metrics->data_sent.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_DATA_SEGS_OUT: {
-        metrics->packet_sent.set(read_unaligned<uint64_t>(val));
+        metrics->packet_sent.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_TOTAL_RETRANS: {
-        metrics->packet_retx.set(read_unaligned<uint64_t>(val));
+        metrics->packet_retx.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_DELIVERED: {
-        metrics->packet_delivered.set(read_unaligned<uint32_t>(val));
+        metrics->packet_delivered.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_DELIVERED_CE: {
-        metrics->packet_delivered_ce.set(read_unaligned<uint32_t>(val));
+        metrics->packet_delivered_ce.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_BYTES_RETRANS: {
-        metrics->data_retx.set(read_unaligned<uint64_t>(val));
+        metrics->data_retx.emplace(read_unaligned<uint64_t>(val));
         break;
       }
       case TCP_NLA_DSACK_DUPS: {
-        metrics->packet_spurious_retx.set(read_unaligned<uint32_t>(val));
+        metrics->packet_spurious_retx.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_REORDERING: {
-        metrics->reordering.set(read_unaligned<uint32_t>(val));
+        metrics->reordering.emplace(read_unaligned<uint32_t>(val));
         break;
       }
       case TCP_NLA_SND_SSTHRESH: {
-        metrics->snd_ssthresh.set(read_unaligned<uint32_t>(val));
+        metrics->snd_ssthresh.emplace(read_unaligned<uint32_t>(val));
         break;
       }
     }

data/src/core/lib/iomgr/error.h

@@ -201,10 +201,10 @@ inline void grpc_error_unref(grpc_error* err) {
 
 // Consumes all the errors in the vector and forms a referencing error from
 // them. If the vector is empty, return GRPC_ERROR_NONE.
-template <size_t N>
-static grpc_error* grpc_error_create_from_vector(
-    const char* file, int line, const char* desc,
-    grpc_core::InlinedVector<grpc_error*, N>* error_list) {
+template <typename VectorType>
+static grpc_error* grpc_error_create_from_vector(const char* file, int line,
+                                                 const char* desc,
+                                                 VectorType* error_list) {
   grpc_error* error = GRPC_ERROR_NONE;
   if (error_list->size() != 0) {
     error = grpc_error_create(file, line, grpc_slice_from_static_string(desc),

data/src/core/lib/iomgr/ev_epollex_linux.cc

@@ -41,11 +41,11 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
 
+#include "absl/container/inlined_vector.h"
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/gpr/spinlock.h"
 #include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gpr/useful.h"
-#include "src/core/lib/gprpp/inlined_vector.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/sync.h"
@@ -190,7 +190,15 @@ struct grpc_fd {
     grpc_iomgr_unregister_object(&iomgr_object);
 
     POLLABLE_UNREF(pollable_obj, "fd_pollable");
-    pollset_fds.clear();
+
+    // To clear out the allocations of pollset_fds, we need to swap its
+    // contents with a newly-constructed (and soon to be destructed) local
+    // variable of its same type. This is because InlinedVector::clear is _not_
+    // guaranteed to actually free up allocations and this is important since
+    // this object doesn't have a conventional destructor.
+    absl::InlinedVector<int, 1> pollset_fds_tmp;
+    pollset_fds_tmp.swap(pollset_fds);
+
     gpr_mu_destroy(&pollable_mu);
     gpr_mu_destroy(&orphan_mu);
 
@@ -232,8 +240,8 @@ struct grpc_fd {
 
   // Protects pollable_obj and pollset_fds.
   gpr_mu pollable_mu;
-  grpc_core::InlinedVector<int, 1> pollset_fds;  // Used in PO_MULTI.
-  pollable* pollable_obj = nullptr;              // Used in PO_FD.
+  absl::InlinedVector<int, 1> pollset_fds;  // Used in PO_MULTI.
+  pollable* pollable_obj = nullptr;         // Used in PO_FD.
 
   grpc_core::LockfreeEvent read_closure;
   grpc_core::LockfreeEvent write_closure;

data/src/core/lib/iomgr/load_file.cc

@@ -53,6 +53,7 @@ grpc_error* grpc_load_file(const char* filename, int add_null_terminator,
       gpr_malloc(contents_size + (add_null_terminator ? 1 : 0)));
   bytes_read = fread(contents, 1, contents_size, file);
   if (bytes_read < contents_size) {
+    gpr_free(contents);
     error = GRPC_OS_ERROR(errno, "fread");
     GPR_ASSERT(ferror(file));
     goto end;

data/src/core/lib/iomgr/tcp_client_posix.cc

@@ -247,13 +247,10 @@ finish:
 grpc_error* grpc_tcp_client_prepare_fd(const grpc_channel_args* channel_args,
                                        const grpc_resolved_address* addr,
                                        grpc_resolved_address* mapped_addr,
-                                       grpc_fd** fdobj) {
+                                       int* fd) {
   grpc_dualstack_mode dsmode;
-  int fd;
   grpc_error* error;
-  char* name;
-  char* addr_str;
-  *fdobj = nullptr;
+  *fd = -1;
   /* Use dualstack sockets where available. Set mapped to v6 or v4 mapped to
      v6. */
   if (!grpc_sockaddr_to_v4mapped(addr, mapped_addr)) {
@@ -261,7 +258,7 @@ grpc_error* grpc_tcp_client_prepare_fd(const grpc_channel_args* channel_args,
     memcpy(mapped_addr, addr, sizeof(*mapped_addr));
   }
   error =
-      grpc_create_dualstack_socket(mapped_addr, SOCK_STREAM, 0, &dsmode, &fd);
+      grpc_create_dualstack_socket(mapped_addr, SOCK_STREAM, 0, &dsmode, fd);
   if (error != GRPC_ERROR_NONE) {
     return error;
   }
@@ -271,29 +268,32 @@ grpc_error* grpc_tcp_client_prepare_fd(const grpc_channel_args* channel_args,
       memcpy(mapped_addr, addr, sizeof(*mapped_addr));
     }
   }
-  if ((error = prepare_socket(mapped_addr, fd, channel_args)) !=
+  if ((error = prepare_socket(mapped_addr, *fd, channel_args)) !=
       GRPC_ERROR_NONE) {
     return error;
   }
-  addr_str = grpc_sockaddr_to_uri(mapped_addr);
-  gpr_asprintf(&name, "tcp-client:%s", addr_str);
-  *fdobj = grpc_fd_create(fd, name, true);
-  gpr_free(name);
-  gpr_free(addr_str);
   return GRPC_ERROR_NONE;
 }
 
 void grpc_tcp_client_create_from_prepared_fd(
-    grpc_pollset_set* interested_parties, grpc_closure* closure, grpc_fd* fdobj,
+    grpc_pollset_set* interested_parties, grpc_closure* closure, const int fd,
     const grpc_channel_args* channel_args, const grpc_resolved_address* addr,
     grpc_millis deadline, grpc_endpoint** ep) {
-  const int fd = grpc_fd_wrapped_fd(fdobj);
   int err;
   async_connect* ac;
   do {
     err = connect(fd, reinterpret_cast<const grpc_sockaddr*>(addr->addr),
                   addr->len);
   } while (err < 0 && errno == EINTR);
+
+  char* name;
+  char* addr_str;
+  addr_str = grpc_sockaddr_to_uri(addr);
+  gpr_asprintf(&name, "tcp-client:%s", addr_str);
+  grpc_fd* fdobj = grpc_fd_create(fd, name, true);
+  gpr_free(name);
+  gpr_free(addr_str);
+
   if (err >= 0) {
     char* addr_str = grpc_sockaddr_to_uri(addr);
     *ep = grpc_tcp_client_create_from_fd(fdobj, channel_args, addr_str);
@@ -340,15 +340,15 @@ static void tcp_connect(grpc_closure* closure, grpc_endpoint** ep,
                         const grpc_resolved_address* addr,
                         grpc_millis deadline) {
   grpc_resolved_address mapped_addr;
-  grpc_fd* fdobj = nullptr;
+  int fd = -1;
   grpc_error* error;
   *ep = nullptr;
   if ((error = grpc_tcp_client_prepare_fd(channel_args, addr, &mapped_addr,
-                                          &fdobj)) != GRPC_ERROR_NONE) {
+                                          &fd)) != GRPC_ERROR_NONE) {
     grpc_core::ExecCtx::Run(DEBUG_LOCATION, closure, error);
     return;
   }
-  grpc_tcp_client_create_from_prepared_fd(interested_parties, closure, fdobj,
+  grpc_tcp_client_create_from_prepared_fd(interested_parties, closure, fd,
                                           channel_args, &mapped_addr, deadline,
                                           ep);
 }

data/src/core/lib/iomgr/tcp_client_posix.h

@@ -35,33 +35,33 @@
 grpc_endpoint* grpc_tcp_client_create_from_fd(
     grpc_fd* fd, const grpc_channel_args* channel_args, const char* addr_str);
 
-/* Return a configured, unbound, unconnected TCP client grpc_fd.
+/* Return a configured, unbound, unconnected TCP client fd.
 
    channel_args: may contain custom settings for the fd
    addr: the destination address
    mapped_addr: out parameter. addr mapped to an address appropriate to the
      type of socket FD created. For example, if addr is IPv4 and dual stack
      sockets are available, mapped_addr will be an IPv4-mapped IPv6 address
-   fdobj: out parameter. The new FD
+   fd: out parameter. The new FD
    Returns: error, if any. Out parameters are not set on error
 */
 grpc_error* grpc_tcp_client_prepare_fd(const grpc_channel_args* channel_args,
                                        const grpc_resolved_address* addr,
                                        grpc_resolved_address* mapped_addr,
-                                       grpc_fd** fdobj);
+                                       int* fd);
 
-/* Connect a configured TCP client grpc_fd.
+/* Connect a configured TCP client fd.
 
    interested_parties: a set of pollsets that would be interested in this
      connection being established (in order to continue their work
    closure: called when complete. On success, *ep will be set.
-   fdobj: an FD returned from grpc_tcp_client_prepare_fd(). Ownership is taken
+   fd: an FD returned from grpc_tcp_client_prepare_fd().
    channel_args: may contain custom settings for the endpoint
    deadline: connection deadline
    ep: out parameter. Set before closure is called if successful
 */
 void grpc_tcp_client_create_from_prepared_fd(
-    grpc_pollset_set* interested_parties, grpc_closure* closure, grpc_fd* fdobj,
+    grpc_pollset_set* interested_parties, grpc_closure* closure, const int fd,
     const grpc_channel_args* channel_args, const grpc_resolved_address* addr,
     grpc_millis deadline, grpc_endpoint** ep);
 

data/src/core/lib/iomgr/tcp_posix.cc

@@ -1385,7 +1385,8 @@ static bool do_tcp_flush_zerocopy(grpc_tcp* tcp, TcpZerocopySendRecord* record,
 
 static void UnrefMaybePutZerocopySendRecord(grpc_tcp* tcp,
                                             TcpZerocopySendRecord* record,
-                                            uint32_t seq, const char* tag) {
+                                            uint32_t seq,
+                                            const char* /* tag */) {
   if (record->Unref()) {
     tcp->tcp_zerocopy_send_ctx.PutSendRecord(record);
   }

data/src/core/lib/iomgr/work_serializer.cc

@@ -0,0 +1,155 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/iomgr/work_serializer.h"
+
+namespace grpc_core {
+
+DebugOnlyTraceFlag grpc_work_serializer_trace(false, "work_serializer");
+
+struct CallbackWrapper {
+  CallbackWrapper(std::function<void()> cb, const grpc_core::DebugLocation& loc)
+      : callback(std::move(cb)), location(loc) {}
+
+  MultiProducerSingleConsumerQueue::Node mpscq_node;
+  const std::function<void()> callback;
+  const DebugLocation location;
+};
+
+class WorkSerializer::WorkSerializerImpl : public Orphanable {
+ public:
+  void Run(std::function<void()> callback,
+           const grpc_core::DebugLocation& location);
+
+  void Orphan() override;
+
+ private:
+  void DrainQueue();
+
+  // An initial size of 1 keeps track of whether the work serializer has been
+  // orphaned.
+  Atomic<size_t> size_{1};
+  MultiProducerSingleConsumerQueue queue_;
+};
+
+void WorkSerializer::WorkSerializerImpl::Run(
+    std::function<void()> callback, const grpc_core::DebugLocation& location) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
+    gpr_log(GPR_INFO, "WorkSerializer::Run() %p Scheduling callback [%s:%d]",
+            this, location.file(), location.line());
+  }
+  const size_t prev_size = size_.FetchAdd(1);
+  // The work serializer should not have been orphaned.
+  GPR_DEBUG_ASSERT(prev_size > 0);
+  if (prev_size == 1) {
+    // There is no other closure executing right now on this work serializer.
+    // Execute this closure immediately.
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
+      gpr_log(GPR_INFO, "  Executing immediately");
+    }
+    callback();
+    // Loan this thread to the work serializer thread and drain the queue.
+    DrainQueue();
+  } else {
+    CallbackWrapper* cb_wrapper =
+        new CallbackWrapper(std::move(callback), location);
+    // There already are closures executing on this work serializer. Simply add
+    // this closure to the queue.
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
+      gpr_log(GPR_INFO, "  Scheduling on queue : item %p", cb_wrapper);
+    }
+    queue_.Push(&cb_wrapper->mpscq_node);
+  }
+}
+
+void WorkSerializer::WorkSerializerImpl::Orphan() {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
+    gpr_log(GPR_INFO, "WorkSerializer::Orphan() %p", this);
+  }
+  size_t prev_size = size_.FetchSub(1);
+  if (prev_size == 1) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
+      gpr_log(GPR_INFO, "  Destroying");
+    }
+    delete this;
+  }
+}
+
+// The thread that calls this loans itself to the work serializer so as to
+// execute all the scheduled callback. This is called from within
+// WorkSerializer::Run() after executing a callback immediately, and hence size_
+// is at least 1.
+void WorkSerializer::WorkSerializerImpl::DrainQueue() {
+  while (true) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
+      gpr_log(GPR_INFO, "WorkSerializer::DrainQueue() %p", this);
+    }
+    size_t prev_size = size_.FetchSub(1);
+    GPR_DEBUG_ASSERT(prev_size >= 1);
+    // It is possible that while draining the queue, one of the callbacks ended
+    // up orphaning the work serializer. In that case, delete the object.
+    if (prev_size == 1) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
+        gpr_log(GPR_INFO, "  Queue Drained. Destroying");
+      }
+      delete this;
+      return;
+    }
+    if (prev_size == 2) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
+        gpr_log(GPR_INFO, "  Queue Drained");
+      }
+      return;
+    }
+    // There is at least one callback on the queue. Pop the callback from the
+    // queue and execute it.
+    CallbackWrapper* cb_wrapper = nullptr;
+    bool empty_unused;
+    while ((cb_wrapper = reinterpret_cast<CallbackWrapper*>(
+                queue_.PopAndCheckEnd(&empty_unused))) == nullptr) {
+      // This can happen either due to a race condition within the mpscq
+      // implementation or because of a race with Run()
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
+        gpr_log(GPR_INFO, "  Queue returned nullptr, trying again");
+      }
+    }
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_work_serializer_trace)) {
+      gpr_log(GPR_INFO, "  Running item %p : callback scheduled at [%s:%d]",
+              cb_wrapper, cb_wrapper->location.file(),
+              cb_wrapper->location.line());
+    }
+    cb_wrapper->callback();
+    delete cb_wrapper;
+  }
+}
+
+// WorkSerializer
+
+WorkSerializer::WorkSerializer()
+    : impl_(MakeOrphanable<WorkSerializerImpl>()) {}
+
+WorkSerializer::~WorkSerializer() {}
+
+void WorkSerializer::Run(std::function<void()> callback,
+                         const grpc_core::DebugLocation& location) {
+  impl_->Run(std::move(callback), location);
+}
+
+}  // namespace grpc_core