grpc 1.20.0 → 1.21.0

data/src/core/lib/slice/slice_buffer.cc

@@ -33,6 +33,10 @@
 #define GROW(x) (3 * (x) / 2)
 
 static void maybe_embiggen(grpc_slice_buffer* sb) {
+  if (sb->count == 0) {
+    sb->slices = sb->base_slices;
+  }
+
   /* How far away from sb->base_slices is sb->slices pointer */
   size_t slice_offset = static_cast<size_t>(sb->slices - sb->base_slices);
   size_t slice_count = sb->count + slice_offset;
@@ -177,6 +181,7 @@ void grpc_slice_buffer_reset_and_unref_internal(grpc_slice_buffer* sb) {
 
   sb->count = 0;
   sb->length = 0;
+  sb->slices = sb->base_slices;
 }
 
 void grpc_slice_buffer_reset_and_unref(grpc_slice_buffer* sb) {

data/src/core/lib/slice/slice_intern.cc

@@ -27,6 +27,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/gpr/murmur_hash.h"
+#include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/iomgr/iomgr_internal.h" /* for iomgr_abort_on_leaks() */
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
@@ -39,24 +40,17 @@
 #define TABLE_IDX(hash, capacity) (((hash) >> LOG2_SHARD_COUNT) % (capacity))
 #define SHARD_IDX(hash) ((hash) & ((1 << LOG2_SHARD_COUNT) - 1))
 
-typedef struct interned_slice_refcount {
-  grpc_slice_refcount base;
-  grpc_slice_refcount sub;
-  size_t length;
-  gpr_atm refcnt;
-  uint32_t hash;
-  struct interned_slice_refcount* bucket_next;
-} interned_slice_refcount;
+using grpc_core::InternedSliceRefcount;
 
 typedef struct slice_shard {
   gpr_mu mu;
-  interned_slice_refcount** strs;
+  InternedSliceRefcount** strs;
   size_t count;
   size_t capacity;
 } slice_shard;
 
 /* hash seed: decided at initialization time */
-static uint32_t g_hash_seed;
+uint32_t g_hash_seed;
 static int g_forced_hash_seed = 0;
 
 static slice_shard g_shards[SHARD_COUNT];
@@ -69,73 +63,35 @@ typedef struct {
 static static_metadata_hash_ent
     static_metadata_hash[4 * GRPC_STATIC_MDSTR_COUNT];
 static uint32_t max_static_metadata_hash_probe;
-static uint32_t static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
+uint32_t grpc_static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
 
-static void interned_slice_ref(void* p) {
-  interned_slice_refcount* s = static_cast<interned_slice_refcount*>(p);
-  GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&s->refcnt, 1) > 0);
-}
+namespace grpc_core {
 
-static void interned_slice_destroy(interned_slice_refcount* s) {
-  slice_shard* shard = &g_shards[SHARD_IDX(s->hash)];
-  gpr_mu_lock(&shard->mu);
-  GPR_ASSERT(0 == gpr_atm_no_barrier_load(&s->refcnt));
-  interned_slice_refcount** prev_next;
-  interned_slice_refcount* cur;
-  for (prev_next = &shard->strs[TABLE_IDX(s->hash, shard->capacity)],
+InternedSliceRefcount::~InternedSliceRefcount() {
+  slice_shard* shard = &g_shards[SHARD_IDX(this->hash)];
+  MutexLock lock(&shard->mu);
+  InternedSliceRefcount** prev_next;
+  InternedSliceRefcount* cur;
+  for (prev_next = &shard->strs[TABLE_IDX(this->hash, shard->capacity)],
       cur = *prev_next;
-       cur != s; prev_next = &cur->bucket_next, cur = cur->bucket_next)
+       cur != this; prev_next = &cur->bucket_next, cur = cur->bucket_next)
     ;
   *prev_next = cur->bucket_next;
   shard->count--;
-  gpr_free(s);
-  gpr_mu_unlock(&shard->mu);
-}
-
-static void interned_slice_unref(void* p) {
-  interned_slice_refcount* s = static_cast<interned_slice_refcount*>(p);
-  if (1 == gpr_atm_full_fetch_add(&s->refcnt, -1)) {
-    interned_slice_destroy(s);
-  }
-}
-
-static void interned_slice_sub_ref(void* p) {
-  interned_slice_ref((static_cast<char*>(p)) -
-                     offsetof(interned_slice_refcount, sub));
-}
-
-static void interned_slice_sub_unref(void* p) {
-  interned_slice_unref((static_cast<char*>(p)) -
-                       offsetof(interned_slice_refcount, sub));
-}
-
-static uint32_t interned_slice_hash(grpc_slice slice) {
-  interned_slice_refcount* s =
-      reinterpret_cast<interned_slice_refcount*>(slice.refcount);
-  return s->hash;
-}
-
-static int interned_slice_eq(grpc_slice a, grpc_slice b) {
-  return a.refcount == b.refcount;
 }
 
-static const grpc_slice_refcount_vtable interned_slice_vtable = {
-    interned_slice_ref, interned_slice_unref, interned_slice_eq,
-    interned_slice_hash};
-static const grpc_slice_refcount_vtable interned_slice_sub_vtable = {
-    interned_slice_sub_ref, interned_slice_sub_unref,
-    grpc_slice_default_eq_impl, grpc_slice_default_hash_impl};
+}  // namespace grpc_core
 
 static void grow_shard(slice_shard* shard) {
   GPR_TIMER_SCOPE("grow_strtab", 0);
 
   size_t capacity = shard->capacity * 2;
   size_t i;
-  interned_slice_refcount** strtab;
-  interned_slice_refcount *s, *next;
+  InternedSliceRefcount** strtab;
+  InternedSliceRefcount *s, *next;
 
-  strtab = static_cast<interned_slice_refcount**>(
-      gpr_zalloc(sizeof(interned_slice_refcount*) * capacity));
+  strtab = static_cast<InternedSliceRefcount**>(
+      gpr_zalloc(sizeof(InternedSliceRefcount*) * capacity));
 
   for (i = 0; i < shard->capacity; i++) {
     for (s = shard->strs[i]; s; s = next) {
@@ -150,7 +106,7 @@ static void grow_shard(slice_shard* shard) {
   shard->capacity = capacity;
 }
 
-static grpc_slice materialize(interned_slice_refcount* s) {
+static grpc_slice materialize(InternedSliceRefcount* s) {
   grpc_slice slice;
   slice.refcount = &s->base;
   slice.data.refcounted.bytes = reinterpret_cast<uint8_t*>(s + 1);
@@ -164,7 +120,7 @@ uint32_t grpc_slice_default_hash_impl(grpc_slice s) {
 }
 
 uint32_t grpc_static_slice_hash(grpc_slice s) {
-  return static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(s)];
+  return grpc_static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(s)];
 }
 
 int grpc_static_slice_eq(grpc_slice a, grpc_slice b) {
@@ -173,7 +129,7 @@ int grpc_static_slice_eq(grpc_slice a, grpc_slice b) {
 
 uint32_t grpc_slice_hash(grpc_slice s) {
   return s.refcount == nullptr ? grpc_slice_default_hash_impl(s)
-                               : s.refcount->vtable->hash(s);
+                               : s.refcount->Hash(s);
 }
 
 grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
@@ -197,8 +153,9 @@ grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
 }
 
 bool grpc_slice_is_interned(const grpc_slice& slice) {
-  return (slice.refcount && slice.refcount->vtable == &interned_slice_vtable) ||
-         GRPC_IS_STATIC_METADATA_STRING(slice);
+  return (slice.refcount &&
+          (slice.refcount->GetType() == grpc_slice_refcount::Type::INTERNED ||
+           GRPC_IS_STATIC_METADATA_STRING(slice)));
 }
 
 grpc_slice grpc_slice_intern(grpc_slice slice) {
@@ -208,6 +165,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) {
   }
 
   uint32_t hash = grpc_slice_hash(slice);
+
   for (uint32_t i = 0; i <= max_static_metadata_hash_probe; i++) {
     static_metadata_hash_ent ent =
         static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)];
@@ -217,7 +175,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) {
     }
   }
 
-  interned_slice_refcount* s;
+  InternedSliceRefcount* s;
   slice_shard* shard = &g_shards[SHARD_IDX(hash)];
 
   gpr_mu_lock(&shard->mu);
@@ -226,14 +184,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) {
   size_t idx = TABLE_IDX(hash, shard->capacity);
   for (s = shard->strs[idx]; s; s = s->bucket_next) {
     if (s->hash == hash && grpc_slice_eq(slice, materialize(s))) {
-      if (gpr_atm_no_barrier_fetch_add(&s->refcnt, 1) == 0) {
-        /* If we get here, we've added a ref to something that was about to
-         * die - drop it immediately.
-         * The *only* possible path here (given the shard mutex) should be to
-         * drop from one ref back to zero - assert that with a CAS */
-        GPR_ASSERT(gpr_atm_rel_cas(&s->refcnt, 1, 0));
-        /* and treat this as if we were never here... sshhh */
-      } else {
+      if (s->refcnt.RefIfNonZero()) {
         gpr_mu_unlock(&shard->mu);
         return materialize(s);
       }
@@ -242,27 +193,20 @@ grpc_slice grpc_slice_intern(grpc_slice slice) {
 
   /* not found: create a new string */
   /* string data goes after the internal_string header */
-  s = static_cast<interned_slice_refcount*>(
+  s = static_cast<InternedSliceRefcount*>(
       gpr_malloc(sizeof(*s) + GRPC_SLICE_LENGTH(slice)));
-  gpr_atm_rel_store(&s->refcnt, 1);
-  s->length = GRPC_SLICE_LENGTH(slice);
-  s->hash = hash;
-  s->base.vtable = &interned_slice_vtable;
-  s->base.sub_refcount = &s->sub;
-  s->sub.vtable = &interned_slice_sub_vtable;
-  s->sub.sub_refcount = &s->sub;
-  s->bucket_next = shard->strs[idx];
-  shard->strs[idx] = s;
-  memcpy(s + 1, GRPC_SLICE_START_PTR(slice), GRPC_SLICE_LENGTH(slice));
 
+  new (s) grpc_core::InternedSliceRefcount(GRPC_SLICE_LENGTH(slice), hash,
+                                           shard->strs[idx]);
+  memcpy(reinterpret_cast<char*>(s + 1), GRPC_SLICE_START_PTR(slice),
+         GRPC_SLICE_LENGTH(slice));
+  shard->strs[idx] = s;
   shard->count++;
-
   if (shard->count > shard->capacity * 2) {
     grow_shard(shard);
   }
 
   gpr_mu_unlock(&shard->mu);
-
   return materialize(s);
 }
 
@@ -280,7 +224,7 @@ void grpc_slice_intern_init(void) {
     gpr_mu_init(&shard->mu);
     shard->count = 0;
     shard->capacity = INITIAL_SHARD_CAPACITY;
-    shard->strs = static_cast<interned_slice_refcount**>(
+    shard->strs = static_cast<InternedSliceRefcount**>(
         gpr_zalloc(sizeof(*shard->strs) * shard->capacity));
   }
   for (size_t i = 0; i < GPR_ARRAY_SIZE(static_metadata_hash); i++) {
@@ -289,13 +233,13 @@ void grpc_slice_intern_init(void) {
   }
   max_static_metadata_hash_probe = 0;
   for (size_t i = 0; i < GRPC_STATIC_MDSTR_COUNT; i++) {
-    static_metadata_hash_values[i] =
+    grpc_static_metadata_hash_values[i] =
         grpc_slice_default_hash_impl(grpc_static_slice_table[i]);
     for (size_t j = 0; j < GPR_ARRAY_SIZE(static_metadata_hash); j++) {
-      size_t slot = (static_metadata_hash_values[i] + j) %
+      size_t slot = (grpc_static_metadata_hash_values[i] + j) %
                     GPR_ARRAY_SIZE(static_metadata_hash);
       if (static_metadata_hash[slot].idx == GRPC_STATIC_MDSTR_COUNT) {
-        static_metadata_hash[slot].hash = static_metadata_hash_values[i];
+        static_metadata_hash[slot].hash = grpc_static_metadata_hash_values[i];
         static_metadata_hash[slot].idx = static_cast<uint32_t>(i);
         if (j > max_static_metadata_hash_probe) {
           max_static_metadata_hash_probe = static_cast<uint32_t>(j);
@@ -315,8 +259,7 @@ void grpc_slice_intern_shutdown(void) {
       gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata strings were leaked",
               shard->count);
       for (size_t j = 0; j < shard->capacity; j++) {
-        for (interned_slice_refcount* s = shard->strs[j]; s;
-             s = s->bucket_next) {
+        for (InternedSliceRefcount* s = shard->strs[j]; s; s = s->bucket_next) {
           char* text =
               grpc_dump_slice(materialize(s), GPR_DUMP_HEX | GPR_DUMP_ASCII);
           gpr_log(GPR_DEBUG, "LEAKED: %s", text);

data/src/core/lib/slice/slice_internal.h

@@ -23,17 +23,215 @@
 
 #include <grpc/slice.h>
 #include <grpc/slice_buffer.h>
+#include <string.h>
+
+#include "src/core/lib/gpr/murmur_hash.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/transport/static_metadata.h"
+
+// Interned slices have specific fast-path operations for hashing. To inline
+// these operations, we need to forward declare them here.
+extern uint32_t grpc_static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
+extern uint32_t g_hash_seed;
+
+// grpc_slice_refcount : A reference count for grpc_slice.
+//
+// Non-inlined grpc_slice objects are refcounted. Historically this was
+// implemented via grpc_slice_refcount, a C-style polymorphic class using a
+// manually managed vtable of operations. Subclasses would define their own
+// vtable; the 'virtual' methods (ref, unref, equals and hash) would simply call
+// the function pointers in the vtable as necessary.
+//
+// Unfortunately, this leads to some inefficiencies in the generated code that
+// can be improved upon. For example, equality checking for interned slices is a
+// simple equality check on the refcount pointer. With the vtable approach, this
+// would translate to roughly the following (high-level) instructions:
+//
+// grpc_slice_equals(slice1, slice2):
+//   load vtable->eq -> eq_func
+//   call eq_func(slice1, slice2)
+//
+// interned_slice_equals(slice1, slice2)
+//   load slice1.ref -> r1
+//   load slice2.ref -> r2
+//   cmp r1, r2 -> retval
+//   ret retval
+//
+// This leads to a function call for a function defined in another translation
+// unit, which imposes memory barriers, which reduces the compiler's ability to
+// optimize (in addition to the added overhead of call/ret). Additionally, it
+// may be harder to reason about branch prediction when we're jumping to
+// essentially arbitrarily provided function pointers.
+//
+// In addition, it is arguable that while virtualization was helpful for
+// Equals()/Hash() methods, that it was fundamentally unnecessary for
+// Ref()/Unref().
+//
+// Instead, grpc_slice_refcount provides the same functionality as the C-style
+// virtual class, but in a de-virtualized manner - Eq(), Hash(), Ref() and
+// Unref() are provided within this header file. Fastpaths for Eq()/Hash()
+// (interned and static metadata slices), as well as the Ref() operation, can
+// all be inlined without any memory barriers.
+//
+// It does this by:
+// 1. Using grpc_core::RefCount<> (header-only) for Ref/Unref. Two special cases
+//    need support: No-op ref/unref (eg. static metadata slices) and stream
+//    slice references (where all the slices share the streamref). This is in
+//    addition to the normal case of '1 slice, 1 ref'.
+//    To support these cases, we explicitly track a nullable pointer to the
+//    underlying RefCount<>. No-op ref/unref is used by checking the pointer for
+//    null, and doing nothing if it is. Both stream slice refs and 'normal'
+//    slices use the same path for Ref/Unref (by targeting the non-null
+//    pointer).
+//
+// 2. introducing the notion of grpc_slice_refcount::Type. This describes if a
+//    slice ref is used by a static metadata slice, an interned slice, or other
+//    slices. We switch on the slice ref type in order to provide fastpaths for
+//    Equals() and Hash().
+//
+// In total, this saves us roughly 1-2% latency for unary calls, with smaller
+// calls benefitting. The effect is present, but not as useful, for larger calls
+// where the cost of sending the data dominates.
+struct grpc_slice_refcount {
+ public:
+  enum class Type {
+    STATIC,    // Refcount for a static metadata slice.
+    INTERNED,  // Refcount for an interned slice.
+    REGULAR    // Refcount for non-static-metadata, non-interned slices.
+  };
+  typedef void (*DestroyerFn)(void*);
+
+  grpc_slice_refcount() = default;
+
+  explicit grpc_slice_refcount(grpc_slice_refcount* sub) : sub_refcount_(sub) {}
+  // Regular constructor for grpc_slice_refcount.
+  //
+  // Parameters:
+  //  1. grpc_slice_refcount::Type type
+  //  Whether we are the refcount for a static
+  //  metadata slice, an interned slice, or any other kind of slice.
+  //
+  //  2. RefCount* ref
+  //  The pointer to the actual underlying grpc_core::RefCount. Rather than
+  //  performing struct offset computations as in the original implementation to
+  //  get to the refcount, which requires a virtual method, we devirtualize by
+  //  using a nullable pointer to allow a single pair of Ref/Unref methods.
+  //
+  //  3. DestroyerFn destroyer_fn
+  //  Called when the refcount goes to 0, with destroyer_arg as parameter.
+  //
+  //  4. void* destroyer_arg
+  //  Argument for the virtualized destructor.
+  //
+  //  5. grpc_slice_refcount* sub
+  //  Argument used for interned slices.
+  grpc_slice_refcount(grpc_slice_refcount::Type type, grpc_core::RefCount* ref,
+                      DestroyerFn destroyer_fn, void* destroyer_arg,
+                      grpc_slice_refcount* sub)
+      : ref_(ref),
+        ref_type_(type),
+        sub_refcount_(sub),
+        dest_fn_(destroyer_fn),
+        destroy_fn_arg_(destroyer_arg) {}
+  // Initializer for static refcounts.
+  grpc_slice_refcount(grpc_slice_refcount* sub, Type type)
+      : ref_type_(type), sub_refcount_(sub) {}
+
+  Type GetType() const { return ref_type_; }
+
+  int Eq(const grpc_slice& a, const grpc_slice& b);
+
+  uint32_t Hash(const grpc_slice& slice);
+  void Ref() {
+    if (ref_ == nullptr) return;
+    ref_->RefNonZero();
+  }
+  void Unref() {
+    if (ref_ == nullptr) return;
+    if (ref_->Unref()) {
+      dest_fn_(destroy_fn_arg_);
+    }
+  }
+
+  grpc_slice_refcount* sub_refcount() const { return sub_refcount_; }
+
+ private:
+  grpc_core::RefCount* ref_ = nullptr;
+  const Type ref_type_ = Type::REGULAR;
+  grpc_slice_refcount* sub_refcount_ = this;
+  DestroyerFn dest_fn_ = nullptr;
+  void* destroy_fn_arg_ = nullptr;
+};
+
+namespace grpc_core {
+
+struct InternedSliceRefcount {
+  static void Destroy(void* arg) {
+    auto* rc = static_cast<InternedSliceRefcount*>(arg);
+    rc->~InternedSliceRefcount();
+    gpr_free(rc);
+  }
+
+  InternedSliceRefcount(size_t length, uint32_t hash,
+                        InternedSliceRefcount* bucket_next)
+      : base(grpc_slice_refcount::Type::INTERNED, &refcnt, Destroy, this, &sub),
+        sub(grpc_slice_refcount::Type::REGULAR, &refcnt, Destroy, this, &sub),
+        length(length),
+        hash(hash),
+        bucket_next(bucket_next) {}
+
+  ~InternedSliceRefcount();
+
+  grpc_slice_refcount base;
+  grpc_slice_refcount sub;
+  const size_t length;
+  RefCount refcnt;
+  const uint32_t hash;
+  InternedSliceRefcount* bucket_next;
+};
+
+}  // namespace grpc_core
+
+inline int grpc_slice_refcount::Eq(const grpc_slice& a, const grpc_slice& b) {
+  switch (ref_type_) {
+    case Type::STATIC:
+      return GRPC_STATIC_METADATA_INDEX(a) == GRPC_STATIC_METADATA_INDEX(b);
+    case Type::INTERNED:
+      return a.refcount == b.refcount;
+    case Type::REGULAR:
+      break;
+  }
+  if (GRPC_SLICE_LENGTH(a) != GRPC_SLICE_LENGTH(b)) return false;
+  if (GRPC_SLICE_LENGTH(a) == 0) return true;
+  return 0 == memcmp(GRPC_SLICE_START_PTR(a), GRPC_SLICE_START_PTR(b),
+                     GRPC_SLICE_LENGTH(a));
+}
+
+inline uint32_t grpc_slice_refcount::Hash(const grpc_slice& slice) {
+  switch (ref_type_) {
+    case Type::STATIC:
+      return ::grpc_static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(
+          slice)];
+    case Type::INTERNED:
+      return reinterpret_cast<grpc_core::InternedSliceRefcount*>(slice.refcount)
+          ->hash;
+    case Type::REGULAR:
+      break;
+  }
+  return gpr_murmur_hash3(GRPC_SLICE_START_PTR(slice), GRPC_SLICE_LENGTH(slice),
+                          g_hash_seed);
+}
 
 inline const grpc_slice& grpc_slice_ref_internal(const grpc_slice& slice) {
   if (slice.refcount) {
-    slice.refcount->vtable->ref(slice.refcount);
+    slice.refcount->Ref();
   }
   return slice;
 }
 
 inline void grpc_slice_unref_internal(const grpc_slice& slice) {
   if (slice.refcount) {
-    slice.refcount->vtable->unref(slice.refcount);
+    slice.refcount->Unref();
   }
 }