grpc 1.23.1 → 1.24.0.pre1

data/src/core/lib/gpr/time_precise.h

@@ -21,9 +21,46 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <grpc/impl/codegen/gpr_types.h>
 #include <grpc/support/time.h>
 
+// Depending on the platform gpr_get_cycle_counter() can have a resolution as
+// low as a usec. Use other clock sources or gpr_precise_clock_now(),
+// where you need high resolution clocks.
+//
+// Using gpr_get_cycle_counter() is preferred to using ExecCtx::Get()->Now()
+// whenever possible.
+
+#if GPR_CYCLE_COUNTER_RDTSC_32
+typedef int64_t gpr_cycle_counter;
+inline gpr_cycle_counter gpr_get_cycle_counter() {
+  int64_t ret;
+  __asm__ volatile("rdtsc" : "=A"(ret));
+  return ret;
+}
+#elif GPR_CYCLE_COUNTER_RDTSC_64
+typedef int64_t gpr_cycle_counter;
+inline gpr_cycle_counter gpr_get_cycle_counter() {
+  uint64_t low, high;
+  __asm__ volatile("rdtsc" : "=a"(low), "=d"(high));
+  return (high << 32) | low;
+}
+#elif GPR_CYCLE_COUNTER_FALLBACK
+// TODO(soheil): add support for mrs on Arm.
+
+// Real time in micros.
+typedef double gpr_cycle_counter;
+gpr_cycle_counter gpr_get_cycle_counter();
+#else
+#error Must define exactly one of \
+    GPR_CYCLE_COUNTER_RDTSC_32, \
+    GPR_CYCLE_COUNTER_RDTSC_64, or \
+    GPR_CYCLE_COUNTER_FALLBACK
+#endif
+
 void gpr_precise_clock_init(void);
 void gpr_precise_clock_now(gpr_timespec* clk);
+gpr_timespec gpr_cycle_counter_to_time(gpr_cycle_counter cycles);
+gpr_timespec gpr_cycle_counter_sub(gpr_cycle_counter a, gpr_cycle_counter b);
 
 #endif /* GRPC_CORE_LIB_GPR_TIME_PRECISE_H */

data/src/core/lib/gprpp/abstract.h

@@ -19,6 +19,14 @@
 #ifndef GRPC_CORE_LIB_GPRPP_ABSTRACT_H
 #define GRPC_CORE_LIB_GPRPP_ABSTRACT_H
 
+#if GRPC_USE_CPP_STD_LIB
+
+#define GRPC_ABSTRACT_BASE_CLASS
+
+#define GRPC_ABSTRACT = 0
+
+#else
+
 // This is needed to support abstract base classes in the c core. Since gRPC
 // doesn't have a c++ runtime, it will hit a linker error on delete unless
 // we define a virtual operator delete. See this blog for more info:
@@ -34,4 +42,6 @@
     GPR_ASSERT(false);                                                       \
   }
 
+#endif  // GRPC_USE_CPP_STD_LIB
+
 #endif /* GRPC_CORE_LIB_GPRPP_ABSTRACT_H */

data/src/core/lib/gprpp/atomic.h

@@ -49,6 +49,10 @@ class Atomic {
     storage_.store(val, static_cast<std::memory_order>(order));
   }
 
+  T Exchange(T desired, MemoryOrder order) {
+    return storage_.exchange(desired, static_cast<std::memory_order>(order));
+  }
+
   bool CompareExchangeWeak(T* expected, T desired, MemoryOrder success,
                            MemoryOrder failure) {
     return GPR_ATM_INC_CAS_THEN(storage_.compare_exchange_weak(

data/src/core/lib/gprpp/inlined_vector.h

@@ -100,16 +100,22 @@ class InlinedVector {
   bool operator==(const InlinedVector& other) const {
     if (size_ != other.size_) return false;
     for (size_t i = 0; i < size_; ++i) {
-      if (data()[i] != other.data()[i]) return false;
+      // Note that this uses == instead of != so that the data class doesn't
+      // have to implement !=.
+      if (!(data()[i] == other.data()[i])) return false;
     }
     return true;
   }
 
   void reserve(size_t capacity) {
     if (capacity > capacity_) {
-      T* new_dynamic = static_cast<T*>(gpr_malloc(sizeof(T) * capacity));
+      T* new_dynamic =
+          std::alignment_of<T>::value == 0
+              ? static_cast<T*>(gpr_malloc(sizeof(T) * capacity))
+              : static_cast<T*>(gpr_malloc_aligned(
+                    sizeof(T) * capacity, std::alignment_of<T>::value));
       move_elements(data(), new_dynamic, size_);
-      gpr_free(dynamic_);
+      free_dynamic();
       dynamic_ = new_dynamic;
       capacity_ = capacity;
     }
@@ -194,7 +200,17 @@ class InlinedVector {
       T& value = data()[i];
       value.~T();
     }
-    gpr_free(dynamic_);
+    free_dynamic();
+  }
+
+  void free_dynamic() {
+    if (dynamic_ != nullptr) {
+      if (std::alignment_of<T>::value == 0) {
+        gpr_free(dynamic_);
+      } else {
+        gpr_free_aligned(dynamic_);
+      }
+    }
   }
 
   typename std::aligned_storage<sizeof(T)>::type inline_[N];

data/src/core/lib/gprpp/map.h

@@ -27,28 +27,48 @@
 #include <functional>
 #include <iterator>
 
+#if GRPC_USE_CPP_STD_LIB
+#include <map>
+#endif
+
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/memory.h"
 #include "src/core/lib/gprpp/pair.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/gprpp/string_view.h"
 
 namespace grpc_core {
+
 struct StringLess {
   bool operator()(const char* a, const char* b) const {
     return strcmp(a, b) < 0;
   }
-  bool operator()(const UniquePtr<char>& k1, const UniquePtr<char>& k2) {
-    return strcmp(k1.get(), k2.get()) < 0;
+  bool operator()(const UniquePtr<char>& a, const UniquePtr<char>& b) const {
+    return strcmp(a.get(), b.get()) < 0;
+  }
+  bool operator()(const StringView& a, const StringView& b) const {
+    const size_t min_size = std::min(a.size(), b.size());
+    int c = strncmp(a.data(), b.data(), min_size);
+    if (c != 0) return c < 0;
+    return a.size() < b.size();
   }
 };
 
 template <typename T>
 struct RefCountedPtrLess {
-  bool operator()(const RefCountedPtr<T>& p1, const RefCountedPtr<T>& p2) {
+  bool operator()(const RefCountedPtr<T>& p1,
+                  const RefCountedPtr<T>& p2) const {
     return p1.get() < p2.get();
   }
 };
 
+#if GRPC_USE_CPP_STD_LIB
+
+template <class Key, class T, class Compare = std::less<Key>>
+using Map = std::map<Key, T, Compare>;
+
+#else  // GRPC_USE_CPP_STD_LIB
+
 namespace testing {
 class MapTest;
 }
@@ -62,6 +82,7 @@ class Map {
   typedef Pair<key_type, mapped_type> value_type;
   typedef Compare key_compare;
   class iterator;
+  class const_iterator;
 
   Map() = default;
   ~Map() { clear(); }
@@ -82,6 +103,22 @@ class Map {
     return *this;
   }
 
+  // Copyable.
+  Map(const Map& other) {
+    for (const auto& p : other) {
+      emplace(p);
+    }
+  }
+  Map& operator=(const Map& other) {
+    if (this != &other) {
+      clear();
+      for (const auto& p : other) {
+        emplace(p);
+      }
+    }
+    return *this;
+  }
+
   T& operator[](key_type&& key);
   T& operator[](const key_type& key);
   iterator find(const key_type& k);
@@ -89,7 +126,7 @@ class Map {
   // Removes the current entry and points to the next one
   iterator erase(iterator iter);
 
-  size_t size() { return size_; }
+  size_t size() const { return size_; }
 
   template <class... Args>
   Pair<iterator, bool> emplace(Args&&... args);
@@ -116,8 +153,19 @@ class Map {
 
   iterator end() { return iterator(this, nullptr); }
 
+  const_iterator begin() const {
+    Entry* curr = GetMinEntry(root_);
+    return const_iterator(this, curr);
+  }
+
+  const_iterator end() const { return const_iterator(this, nullptr); }
+
   iterator lower_bound(const Key& k) {
-    key_compare compare;
+    // This is a workaround for "const key_compare compare;"
+    // because some versions of compilers cannot build this by requiring
+    // a user-provided constructor. (ref: https://stackoverflow.com/q/7411515)
+    key_compare compare_tmp;
+    const key_compare& compare = compare_tmp;
     return std::find_if(begin(), end(), [&k, &compare](const value_type& v) {
       return !compare(v.first, k);
     });
@@ -204,6 +252,53 @@ class Map<Key, T, Compare>::iterator
   GrpcMap* map_;
 };
 
+template <class Key, class T, class Compare>
+class Map<Key, T, Compare>::const_iterator
+    : public std::iterator<std::input_iterator_tag, Pair<Key, T>, int32_t,
+                           Pair<Key, T>*, Pair<Key, T>&> {
+ public:
+  const_iterator(const const_iterator& iter)
+      : curr_(iter.curr_), map_(iter.map_) {}
+  bool operator==(const const_iterator& rhs) const {
+    return (curr_ == rhs.curr_);
+  }
+  bool operator!=(const const_iterator& rhs) const {
+    return (curr_ != rhs.curr_);
+  }
+
+  const_iterator& operator++() {
+    curr_ = map_->InOrderSuccessor(curr_);
+    return *this;
+  }
+
+  const_iterator operator++(int) {
+    Entry* prev = curr_;
+    curr_ = map_->InOrderSuccessor(curr_);
+    return const_iterator(map_, prev);
+  }
+
+  const_iterator& operator=(const const_iterator& other) {
+    if (this != &other) {
+      this->curr_ = other.curr_;
+      this->map_ = other.map_;
+    }
+    return *this;
+  }
+
+  // operator*()
+  const value_type& operator*() const { return curr_->pair; }
+
+  // operator->()
+  const value_type* operator->() const { return &curr_->pair; }
+
+ private:
+  friend class Map<key_type, mapped_type, key_compare>;
+  using GrpcMap = typename ::grpc_core::Map<Key, T, Compare>;
+  const_iterator(const GrpcMap* map, Entry* curr) : curr_(curr), map_(map) {}
+  Entry* curr_;
+  const GrpcMap* map_;
+};
+
 template <class Key, class T, class Compare>
 T& Map<Key, T, Compare>::operator[](key_type&& key) {
   auto iter = find(key);
@@ -448,7 +543,11 @@ Map<Key, T, Compare>::RemoveRecursive(Entry* root, const key_type& k) {
 template <class Key, class T, class Compare>
 int Map<Key, T, Compare>::CompareKeys(const key_type& lhs,
                                       const key_type& rhs) {
-  key_compare compare;
+  // This is a workaround for "const key_compare compare;"
+  // because some versions of compilers cannot build this by requiring
+  // a user-provided constructor. (ref: https://stackoverflow.com/q/7411515)
+  key_compare compare_tmp;
+  const key_compare& compare = compare_tmp;
   bool left_comparison = compare(lhs, rhs);
   bool right_comparison = compare(rhs, lhs);
   // Both values are equal
@@ -457,5 +556,9 @@ int Map<Key, T, Compare>::CompareKeys(const key_type& lhs,
   }
   return left_comparison ? -1 : 1;
 }
+
+#endif  // GRPC_USE_CPP_STD_LIB
+
 }  // namespace grpc_core
+
 #endif /* GRPC_CORE_LIB_GPRPP_MAP_H */

data/src/core/lib/gprpp/memory.h

@@ -30,6 +30,9 @@
 
 // Add this to a class that want to use Delete(), but has a private or
 // protected destructor.
+// Should not be used in new code.
+// TODO(juanlishen): Remove this macro, and instead comment that the public dtor
+// should not be used directly.
 #define GRPC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE         \
   template <typename _Delete_T, bool _Delete_can_be_null> \
   friend void ::grpc_core::Delete(_Delete_T*);            \
@@ -38,6 +41,9 @@
 
 // Add this to a class that want to use New(), but has a private or
 // protected constructor.
+// Should not be used in new code.
+// TODO(juanlishen): Remove this macro, and instead comment that the public dtor
+// should not be used directly.
 #define GRPC_ALLOW_CLASS_TO_USE_NON_PUBLIC_NEW      \
   template <typename _New_T, typename... _New_Args> \
   friend _New_T* grpc_core::New(_New_Args&&...);

data/src/core/lib/gprpp/ref_counted_ptr.h

@@ -103,6 +103,8 @@ class RefCountedPtr {
     if (value_ != nullptr) value_->Unref();
   }
 
+  void swap(RefCountedPtr& other) { std::swap(value_, other.value_); }
+
   // If value is non-null, we take ownership of a ref to it.
   void reset(T* value = nullptr) {
     if (value_ != nullptr) value_->Unref();

data/src/core/lib/iomgr/ev_epollex_linux.cc

@@ -47,6 +47,7 @@
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/inlined_vector.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
+#include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
@@ -89,7 +90,7 @@ typedef struct pollable pollable;
 ///  - PO_MULTI - a pollable containing many fds
 struct pollable {
   pollable_type type;  // immutable
-  gpr_refcount refs;
+  grpc_core::RefCount refs;
 
   int epfd;
   grpc_wakeup_fd wakeup;
@@ -135,17 +136,26 @@ static char* pollable_desc(pollable* p) {
 static pollable* g_empty_pollable;
 
 static grpc_error* pollable_create(pollable_type type, pollable** p);
-#ifdef NDEBUG
-static pollable* pollable_ref(pollable* p);
-static void pollable_unref(pollable* p);
-#define POLLABLE_REF(p, r) pollable_ref(p)
-#define POLLABLE_UNREF(p, r) pollable_unref(p)
-#else
-static pollable* pollable_ref(pollable* p, int line, const char* reason);
-static void pollable_unref(pollable* p, int line, const char* reason);
-#define POLLABLE_REF(p, r) pollable_ref((p), __LINE__, (r))
-#define POLLABLE_UNREF(p, r) pollable_unref((p), __LINE__, (r))
-#endif
+static pollable* pollable_ref(pollable* p,
+                              const grpc_core::DebugLocation& dbg_loc,
+                              const char* reason) {
+  p->refs.Ref(dbg_loc, reason);
+  return p;
+}
+static void pollable_unref(pollable* p, const grpc_core::DebugLocation& dbg_loc,
+                           const char* reason) {
+  if (p == nullptr) return;
+  if (GPR_UNLIKELY(p != nullptr && p->refs.Unref(dbg_loc, reason))) {
+    GRPC_FD_TRACE("pollable_unref: Closing epfd: %d", p->epfd);
+    close(p->epfd);
+    grpc_wakeup_fd_destroy(&p->wakeup);
+    gpr_mu_destroy(&p->owner_orphan_mu);
+    gpr_mu_destroy(&p->mu);
+    gpr_free(p);
+  }
+}
+#define POLLABLE_REF(p, r) pollable_ref((p), DEBUG_LOCATION, (r))
+#define POLLABLE_UNREF(p, r) pollable_unref((p), DEBUG_LOCATION, (r))
 
 /*******************************************************************************
  * Fd Declarations
@@ -283,7 +293,7 @@ struct grpc_pollset {
  */
 
 struct grpc_pollset_set {
-  gpr_refcount refs;
+  grpc_core::RefCount refs;
   gpr_mu mu;
   grpc_pollset_set* parent;
 
@@ -568,7 +578,7 @@ static grpc_error* pollable_create(pollable_type type, pollable** p) {
   }
 
   (*p)->type = type;
-  gpr_ref_init(&(*p)->refs, 1);
+  new (&(*p)->refs) grpc_core::RefCount(1, &grpc_trace_pollable_refcount);
   gpr_mu_init(&(*p)->mu);
   (*p)->epfd = epfd;
   (*p)->owner_fd = nullptr;
@@ -582,41 +592,6 @@ static grpc_error* pollable_create(pollable_type type, pollable** p) {
   return GRPC_ERROR_NONE;
 }
 
-#ifdef NDEBUG
-static pollable* pollable_ref(pollable* p) {
-#else
-static pollable* pollable_ref(pollable* p, int line, const char* reason) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_pollable_refcount)) {
-    int r = static_cast<int> gpr_atm_no_barrier_load(&p->refs.count);
-    gpr_log(__FILE__, line, GPR_LOG_SEVERITY_DEBUG,
-            "POLLABLE:%p   ref %d->%d %s", p, r, r + 1, reason);
-  }
-#endif
-  gpr_ref(&p->refs);
-  return p;
-}
-
-#ifdef NDEBUG
-static void pollable_unref(pollable* p) {
-#else
-static void pollable_unref(pollable* p, int line, const char* reason) {
-  if (p == nullptr) return;
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_pollable_refcount)) {
-    int r = static_cast<int> gpr_atm_no_barrier_load(&p->refs.count);
-    gpr_log(__FILE__, line, GPR_LOG_SEVERITY_DEBUG,
-            "POLLABLE:%p unref %d->%d %s", p, r, r - 1, reason);
-  }
-#endif
-  if (p != nullptr && gpr_unref(&p->refs)) {
-    GRPC_FD_TRACE("pollable_unref: Closing epfd: %d", p->epfd);
-    close(p->epfd);
-    grpc_wakeup_fd_destroy(&p->wakeup);
-    gpr_mu_destroy(&p->owner_orphan_mu);
-    gpr_mu_destroy(&p->mu);
-    gpr_free(p);
-  }
-}
-
 static grpc_error* pollable_add_fd(pollable* p, grpc_fd* fd) {
   grpc_error* error = GRPC_ERROR_NONE;
   static const char* err_desc = "pollable_add_fd";
@@ -1102,7 +1077,7 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker,
 }
 
 #ifndef NDEBUG
-static long gettid(void) { return syscall(__NR_gettid); }
+static long sys_gettid(void) { return syscall(__NR_gettid); }
 #endif
 
 /* pollset->mu lock must be held by the caller before calling this.
@@ -1122,7 +1097,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset,
 #define WORKER_PTR (&worker)
 #endif
 #ifndef NDEBUG
-  WORKER_PTR->originator = gettid();
+  WORKER_PTR->originator = sys_gettid();
 #endif
   if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) {
     gpr_log(GPR_INFO,
@@ -1331,13 +1306,13 @@ static grpc_pollset_set* pollset_set_create(void) {
   grpc_pollset_set* pss =
       static_cast<grpc_pollset_set*>(gpr_zalloc(sizeof(*pss)));
   gpr_mu_init(&pss->mu);
-  gpr_ref_init(&pss->refs, 1);
+  new (&pss->refs) grpc_core::RefCount();
   return pss;
 }
 
 static void pollset_set_unref(grpc_pollset_set* pss) {
   if (pss == nullptr) return;
-  if (!gpr_unref(&pss->refs)) return;
+  if (GPR_LIKELY(!pss->refs.Unref())) return;
   pollset_set_unref(pss->parent);
   gpr_mu_destroy(&pss->mu);
   for (size_t i = 0; i < pss->pollset_count; i++) {
@@ -1528,7 +1503,7 @@ static void pollset_set_add_pollset_set(grpc_pollset_set* a,
   if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) {
     gpr_log(GPR_INFO, "PSS: parent %p to %p", b, a);
   }
-  gpr_ref(&a->refs);
+  a->refs.Ref();
   b->parent = a;
   if (a->fd_capacity < a->fd_count + b->fd_count) {
     a->fd_capacity = GPR_MAX(2 * a->fd_capacity, a->fd_count + b->fd_count);