grpc 1.21.0 → 1.22.0

data/src/core/lib/slice/slice_buffer.cc

@@ -32,35 +32,45 @@
 /* grow a buffer; requires GRPC_SLICE_BUFFER_INLINE_ELEMENTS > 1 */
 #define GROW(x) (3 * (x) / 2)
 
+/* Typically, we do not actually need to embiggen (by calling
+ * memmove/malloc/realloc) - only if we were up against the full capacity of the
+ * slice buffer. If do_embiggen is inlined, the compiler clobbers multiple
+ * registers pointlessly in the common case. */
+static void GPR_ATTRIBUTE_NOINLINE do_embiggen(grpc_slice_buffer* sb,
+                                               const size_t slice_count,
+                                               const size_t slice_offset) {
+  if (slice_offset != 0) {
+    /* Make room by moving elements if there's still space unused */
+    memmove(sb->base_slices, sb->slices, sb->count * sizeof(grpc_slice));
+    sb->slices = sb->base_slices;
+  } else {
+    /* Allocate more memory if no more space is available */
+    const size_t new_capacity = GROW(sb->capacity);
+    sb->capacity = new_capacity;
+    if (sb->base_slices == sb->inlined) {
+      sb->base_slices = static_cast<grpc_slice*>(
+          gpr_malloc(new_capacity * sizeof(grpc_slice)));
+      memcpy(sb->base_slices, sb->inlined, slice_count * sizeof(grpc_slice));
+    } else {
+      sb->base_slices = static_cast<grpc_slice*>(
+          gpr_realloc(sb->base_slices, new_capacity * sizeof(grpc_slice)));
+    }
+
+    sb->slices = sb->base_slices + slice_offset;
+  }
+}
+
 static void maybe_embiggen(grpc_slice_buffer* sb) {
   if (sb->count == 0) {
     sb->slices = sb->base_slices;
+    return;
   }
 
   /* How far away from sb->base_slices is sb->slices pointer */
   size_t slice_offset = static_cast<size_t>(sb->slices - sb->base_slices);
   size_t slice_count = sb->count + slice_offset;
-
-  if (slice_count == sb->capacity) {
-    if (sb->base_slices != sb->slices) {
-      /* Make room by moving elements if there's still space unused */
-      memmove(sb->base_slices, sb->slices, sb->count * sizeof(grpc_slice));
-      sb->slices = sb->base_slices;
-    } else {
-      /* Allocate more memory if no more space is available */
-      sb->capacity = GROW(sb->capacity);
-      GPR_ASSERT(sb->capacity > slice_count);
-      if (sb->base_slices == sb->inlined) {
-        sb->base_slices = static_cast<grpc_slice*>(
-            gpr_malloc(sb->capacity * sizeof(grpc_slice)));
-        memcpy(sb->base_slices, sb->inlined, slice_count * sizeof(grpc_slice));
-      } else {
-        sb->base_slices = static_cast<grpc_slice*>(
-            gpr_realloc(sb->base_slices, sb->capacity * sizeof(grpc_slice)));
-      }
-
-      sb->slices = sb->base_slices + slice_offset;
-    }
+  if (GPR_UNLIKELY(slice_count == sb->capacity)) {
+    do_embiggen(sb, slice_count, slice_offset);
   }
 }
 
@@ -375,6 +385,24 @@ grpc_slice grpc_slice_buffer_take_first(grpc_slice_buffer* sb) {
   return slice;
 }
 
+void grpc_slice_buffer_remove_first(grpc_slice_buffer* sb) {
+  GPR_DEBUG_ASSERT(sb->count > 0);
+  sb->length -= GRPC_SLICE_LENGTH(sb->slices[0]);
+  grpc_slice_unref_internal(sb->slices[0]);
+  sb->slices++;
+  if (--sb->count == 0) {
+    sb->slices = sb->base_slices;
+  }
+}
+
+void grpc_slice_buffer_sub_first(grpc_slice_buffer* sb, size_t begin,
+                                 size_t end) {
+  // TODO(soheil): Introduce a ptr version for sub.
+  sb->length -= GRPC_SLICE_LENGTH(sb->slices[0]);
+  sb->slices[0] = grpc_slice_sub_no_ref(sb->slices[0], begin, end);
+  sb->length += end - begin;
+}
+
 void grpc_slice_buffer_undo_take_first(grpc_slice_buffer* sb,
                                        grpc_slice slice) {
   sb->slices--;

data/src/core/lib/slice/slice_hash_table.h

@@ -138,7 +138,7 @@ SliceHashTable<T>::~SliceHashTable() {
 
 template <typename T>
 void SliceHashTable<T>::Add(const grpc_slice& key, T& value) {
-  const size_t hash = grpc_slice_hash(key);
+  const size_t hash = grpc_slice_hash_internal(key);
   for (size_t offset = 0; offset < size_; ++offset) {
     const size_t idx = (hash + offset) % size_;
     if (!entries_[idx].is_set) {
@@ -156,7 +156,7 @@ void SliceHashTable<T>::Add(const grpc_slice& key, T& value) {
 
 template <typename T>
 const T* SliceHashTable<T>::Get(const grpc_slice& key) const {
-  const size_t hash = grpc_slice_hash(key);
+  const size_t hash = grpc_slice_hash_internal(key);
   // We cap the number of probes at the max number recorded when
   // populating the table.
   for (size_t offset = 0; offset <= max_num_probes_; ++offset) {

data/src/core/lib/slice/slice_intern.cc

@@ -19,6 +19,7 @@
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/slice/slice_utils.h"
 
 #include <inttypes.h>
 #include <string.h>
@@ -127,10 +128,7 @@ int grpc_static_slice_eq(grpc_slice a, grpc_slice b) {
   return GRPC_STATIC_METADATA_INDEX(a) == GRPC_STATIC_METADATA_INDEX(b);
 }
 
-uint32_t grpc_slice_hash(grpc_slice s) {
-  return s.refcount == nullptr ? grpc_slice_default_hash_impl(s)
-                               : s.refcount->Hash(s);
-}
+uint32_t grpc_slice_hash(grpc_slice s) { return grpc_slice_hash_internal(s); }
 
 grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
                                           bool* returned_slice_is_different) {
@@ -138,12 +136,13 @@ grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
     return slice;
   }
 
-  uint32_t hash = grpc_slice_hash(slice);
+  uint32_t hash = grpc_slice_hash_internal(slice);
   for (uint32_t i = 0; i <= max_static_metadata_hash_probe; i++) {
     static_metadata_hash_ent ent =
         static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)];
     if (ent.hash == hash && ent.idx < GRPC_STATIC_MDSTR_COUNT &&
-        grpc_slice_eq(grpc_static_slice_table[ent.idx], slice)) {
+        grpc_slice_eq_static_interned(slice,
+                                      grpc_static_slice_table[ent.idx])) {
       *returned_slice_is_different = true;
       return grpc_static_slice_table[ent.idx];
     }
@@ -152,25 +151,20 @@ grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
   return slice;
 }
 
-bool grpc_slice_is_interned(const grpc_slice& slice) {
-  return (slice.refcount &&
-          (slice.refcount->GetType() == grpc_slice_refcount::Type::INTERNED ||
-           GRPC_IS_STATIC_METADATA_STRING(slice)));
-}
-
 grpc_slice grpc_slice_intern(grpc_slice slice) {
   GPR_TIMER_SCOPE("grpc_slice_intern", 0);
   if (GRPC_IS_STATIC_METADATA_STRING(slice)) {
     return slice;
   }
 
-  uint32_t hash = grpc_slice_hash(slice);
+  uint32_t hash = grpc_slice_hash_internal(slice);
 
   for (uint32_t i = 0; i <= max_static_metadata_hash_probe; i++) {
     static_metadata_hash_ent ent =
         static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)];
     if (ent.hash == hash && ent.idx < GRPC_STATIC_MDSTR_COUNT &&
-        grpc_slice_eq(grpc_static_slice_table[ent.idx], slice)) {
+        grpc_slice_eq_static_interned(slice,
+                                      grpc_static_slice_table[ent.idx])) {
       return grpc_static_slice_table[ent.idx];
     }
   }
@@ -183,7 +177,8 @@ grpc_slice grpc_slice_intern(grpc_slice slice) {
   /* search for an existing string */
   size_t idx = TABLE_IDX(hash, shard->capacity);
   for (s = shard->strs[idx]; s; s = s->bucket_next) {
-    if (s->hash == hash && grpc_slice_eq(slice, materialize(s))) {
+    if (s->hash == hash &&
+        grpc_slice_eq_static_interned(slice, materialize(s))) {
       if (s->refcnt.RefIfNonZero()) {
         gpr_mu_unlock(&shard->mu);
         return materialize(s);
@@ -234,7 +229,7 @@ void grpc_slice_intern_init(void) {
   max_static_metadata_hash_probe = 0;
   for (size_t i = 0; i < GRPC_STATIC_MDSTR_COUNT; i++) {
     grpc_static_metadata_hash_values[i] =
-        grpc_slice_default_hash_impl(grpc_static_slice_table[i]);
+        grpc_slice_default_hash_internal(grpc_static_slice_table[i]);
     for (size_t j = 0; j < GPR_ARRAY_SIZE(static_metadata_hash); j++) {
       size_t slot = (grpc_static_metadata_hash_values[i] + j) %
                     GPR_ARRAY_SIZE(static_metadata_hash);
@@ -248,6 +243,10 @@ void grpc_slice_intern_init(void) {
       }
     }
   }
+  // Handle KV hash for all static mdelems.
+  for (size_t i = 0; i < GRPC_STATIC_MDELEM_COUNT; ++i) {
+    grpc_static_mdelem_table[i].HashInit();
+  }
 }
 
 void grpc_slice_intern_shutdown(void) {

data/src/core/lib/slice/slice_internal.h

@@ -21,11 +21,14 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <grpc/support/log.h>
+
 #include <grpc/slice.h>
 #include <grpc/slice_buffer.h>
 #include <string.h>
 
 #include "src/core/lib/gpr/murmur_hash.h"
+#include "src/core/lib/gprpp/memory.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/transport/static_metadata.h"
 
@@ -97,12 +100,15 @@ struct grpc_slice_refcount {
   enum class Type {
     STATIC,    // Refcount for a static metadata slice.
     INTERNED,  // Refcount for an interned slice.
+    NOP,       // No-Op
     REGULAR    // Refcount for non-static-metadata, non-interned slices.
   };
   typedef void (*DestroyerFn)(void*);
 
   grpc_slice_refcount() = default;
 
+  explicit grpc_slice_refcount(Type t) : ref_type_(t) {}
+
   explicit grpc_slice_refcount(grpc_slice_refcount* sub) : sub_refcount_(sub) {}
   // Regular constructor for grpc_slice_refcount.
   //
@@ -198,6 +204,7 @@ inline int grpc_slice_refcount::Eq(const grpc_slice& a, const grpc_slice& b) {
       return GRPC_STATIC_METADATA_INDEX(a) == GRPC_STATIC_METADATA_INDEX(b);
     case Type::INTERNED:
       return a.refcount == b.refcount;
+    case Type::NOP:
     case Type::REGULAR:
       break;
   }
@@ -215,6 +222,7 @@ inline uint32_t grpc_slice_refcount::Hash(const grpc_slice& slice) {
     case Type::INTERNED:
       return reinterpret_cast<grpc_core::InternedSliceRefcount*>(slice.refcount)
           ->hash;
+    case Type::NOP:
     case Type::REGULAR:
       break;
   }
@@ -240,8 +248,33 @@ void grpc_slice_buffer_partial_unref_internal(grpc_slice_buffer* sb,
                                               size_t idx);
 void grpc_slice_buffer_destroy_internal(grpc_slice_buffer* sb);
 
+// Returns a pointer to the first slice in the slice buffer without giving
+// ownership to or a reference count on that slice.
+inline grpc_slice* grpc_slice_buffer_peek_first(grpc_slice_buffer* sb) {
+  GPR_DEBUG_ASSERT(sb->count > 0);
+  return &sb->slices[0];
+}
+
+// Removes the first slice from the slice buffer.
+void grpc_slice_buffer_remove_first(grpc_slice_buffer* sb);
+
+// Calls grpc_slice_sub with the given parameters on the first slice.
+void grpc_slice_buffer_sub_first(grpc_slice_buffer* sb, size_t begin,
+                                 size_t end);
+
 /* Check if a slice is interned */
 bool grpc_slice_is_interned(const grpc_slice& slice);
+inline bool grpc_slice_is_interned(const grpc_slice& slice) {
+  return (slice.refcount &&
+          (slice.refcount->GetType() == grpc_slice_refcount::Type::INTERNED ||
+           slice.refcount->GetType() == grpc_slice_refcount::Type::STATIC));
+}
+
+inline bool grpc_slice_static_interned_equal(const grpc_slice& a,
+                                             const grpc_slice& b) {
+  GPR_DEBUG_ASSERT(grpc_slice_is_interned(a) && grpc_slice_is_interned(b));
+  return a.refcount == b.refcount;
+}
 
 void grpc_slice_intern_init(void);
 void grpc_slice_intern_shutdown(void);
@@ -255,6 +288,25 @@ grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
 uint32_t grpc_static_slice_hash(grpc_slice s);
 int grpc_static_slice_eq(grpc_slice a, grpc_slice b);
 
+inline uint32_t grpc_slice_hash_refcounted(const grpc_slice& s) {
+  GPR_DEBUG_ASSERT(s.refcount != nullptr);
+  return s.refcount->Hash(s);
+}
+
+inline uint32_t grpc_slice_default_hash_internal(const grpc_slice& s) {
+  return gpr_murmur_hash3(GRPC_SLICE_START_PTR(s), GRPC_SLICE_LENGTH(s),
+                          g_hash_seed);
+}
+
+inline uint32_t grpc_slice_hash_internal(const grpc_slice& s) {
+  return s.refcount == nullptr ? grpc_slice_default_hash_internal(s)
+                               : grpc_slice_hash_refcounted(s);
+}
+
+grpc_slice grpc_slice_from_moved_buffer(grpc_core::UniquePtr<char> p,
+                                        size_t len);
+grpc_slice grpc_slice_from_moved_string(grpc_core::UniquePtr<char> p);
+
 // Returns the memory used by this slice, not counting the slice structure
 // itself. This means that inlined and slices from static strings will return
 // 0. All other slices will return the size of the allocated chars.

data/src/core/lib/slice/slice_string_helpers.cc

@@ -25,13 +25,22 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gprpp/memory.h"
 #include "src/core/lib/slice/slice_internal.h"
 
-char* grpc_dump_slice(grpc_slice s, uint32_t flags) {
+char* grpc_dump_slice(const grpc_slice& s, uint32_t flags) {
   return gpr_dump(reinterpret_cast<const char*> GRPC_SLICE_START_PTR(s),
                   GRPC_SLICE_LENGTH(s), flags);
 }
 
+grpc_slice grpc_dump_slice_to_slice(const grpc_slice& s, uint32_t flags) {
+  size_t len;
+  grpc_core::UniquePtr<char> ptr(
+      gpr_dump_return_len(reinterpret_cast<const char*> GRPC_SLICE_START_PTR(s),
+                          GRPC_SLICE_LENGTH(s), flags, &len));
+  return grpc_slice_from_moved_buffer(std::move(ptr), len);
+}
+
 /** Finds the initial (\a begin) and final (\a end) offsets of the next
  * substring from \a str + \a read_offset until the next \a sep or the end of \a
  * str.

data/src/core/lib/slice/slice_string_helpers.h

@@ -30,7 +30,9 @@
 #include "src/core/lib/gpr/string.h"
 
 /* Calls gpr_dump on a slice. */
-char* grpc_dump_slice(grpc_slice slice, uint32_t flags);
+char* grpc_dump_slice(const grpc_slice& slice, uint32_t flags);
+/* Calls gpr_dump on a slice and returns the result as a slice. */
+grpc_slice grpc_dump_slice_to_slice(const grpc_slice& slice, uint32_t flags);
 
 /** Split \a str by the separator \a sep. Results are stored in \a dst, which
  * should be a properly initialized instance. */

data/src/core/lib/slice/slice_utils.h

@@ -0,0 +1,50 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SLICE_SLICE_UTILS_H
+#define GRPC_CORE_LIB_SLICE_SLICE_UTILS_H
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/slice.h>
+
+// When we compare two slices, and we know the latter is not inlined, we can
+// short circuit our comparison operator. We specifically use differs()
+// semantics instead of equals() semantics due to more favourable code
+// generation when using differs(). Specifically, we may use the output of
+// grpc_slice_differs_refcounted for control flow. If we use differs()
+// semantics, we end with a tailcall to memcmp(). If we use equals() semantics,
+// we need to invert the result that memcmp provides us, which costs several
+// instructions to do so. If we're using the result for control flow (i.e.
+// branching based on the output) then we're just performing the extra
+// operations to invert the result pointlessly. Concretely, we save 6 ops on
+// x86-64/clang with differs().
+int grpc_slice_differs_refcounted(const grpc_slice& a,
+                                  const grpc_slice& b_not_inline);
+// When we compare two slices, and we *know* that one of them is static or
+// interned, we can short circuit our slice equality function. The second slice
+// here must be static or interned; slice a can be any slice, inlined or not.
+inline bool grpc_slice_eq_static_interned(const grpc_slice& a,
+                                          const grpc_slice& b_static_interned) {
+  if (a.refcount == b_static_interned.refcount) {
+    return true;
+  }
+  return !grpc_slice_differs_refcounted(a, b_static_interned);
+}
+
+#endif /* GRPC_CORE_LIB_SLICE_SLICE_UTILS_H */

data/src/core/lib/slice/slice_weak_hash_table.h

@@ -47,7 +47,7 @@ class SliceWeakHashTable : public RefCounted<SliceWeakHashTable<T, Size>> {
   /// Add a mapping from \a key to \a value, taking ownership of \a key. This
   /// operation will always succeed. It may discard older entries.
   void Add(const grpc_slice& key, T value) {
-    const size_t idx = grpc_slice_hash(key) % Size;
+    const size_t idx = grpc_slice_hash_internal(key) % Size;
     entries_[idx].Set(key, std::move(value));
     return;
   }
@@ -55,7 +55,7 @@ class SliceWeakHashTable : public RefCounted<SliceWeakHashTable<T, Size>> {
   /// Returns the value from the table associated with / \a key or null if not
   /// found.
   const T* Get(const grpc_slice& key) const {
-    const size_t idx = grpc_slice_hash(key) % Size;
+    const size_t idx = grpc_slice_hash_internal(key) % Size;
     const auto& entry = entries_[idx];
     return grpc_slice_eq(entry.key(), key) ? entry.value() : nullptr;
   }

data/src/core/lib/surface/call.cc

@@ -42,8 +42,8 @@
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
+#include "src/core/lib/slice/slice_utils.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/call_test_only.h"
@@ -74,7 +74,7 @@
 #define ESTIMATED_MDELEM_COUNT 16
 
 struct batch_control {
-  batch_control() { gpr_ref_init(&steps_to_complete, 0); }
+  batch_control() = default;
 
   grpc_call* call = nullptr;
   grpc_transport_stream_op_batch op;
@@ -99,8 +99,14 @@ struct batch_control {
   } completion_data;
   grpc_closure start_batch;
   grpc_closure finish_batch;
-  gpr_refcount steps_to_complete;
+  grpc_core::Atomic<intptr_t> steps_to_complete;
   gpr_atm batch_error = reinterpret_cast<gpr_atm>(GRPC_ERROR_NONE);
+  void set_num_steps_to_complete(uintptr_t steps) {
+    steps_to_complete.Store(steps, grpc_core::MemoryOrder::RELEASE);
+  }
+  bool completed_batch_step() {
+    return steps_to_complete.FetchSub(1, grpc_core::MemoryOrder::ACQ_REL) == 1;
+  }
 };
 
 struct parent_call {
@@ -349,8 +355,8 @@ grpc_error* grpc_call_create(const grpc_call_create_args* args,
                MAX_SEND_EXTRA_METADATA_COUNT);
     for (size_t i = 0; i < args->add_initial_metadata_count; i++) {
       call->send_extra_metadata[i].md = args->add_initial_metadata[i];
-      if (grpc_slice_eq(GRPC_MDKEY(args->add_initial_metadata[i]),
-                        GRPC_MDSTR_PATH)) {
+      if (grpc_slice_eq_static_interned(
+              GRPC_MDKEY(args->add_initial_metadata[i]), GRPC_MDSTR_PATH)) {
         path = grpc_slice_ref_internal(
             GRPC_MDVALUE(args->add_initial_metadata[i]));
       }
@@ -899,7 +905,7 @@ static int prepare_application_metadata(grpc_call* call, int count,
     if (!GRPC_LOG_IF_ERROR("validate_metadata",
                            grpc_validate_header_key_is_legal(md->key))) {
       break;
-    } else if (!grpc_is_binary_header(md->key) &&
+    } else if (!grpc_is_binary_header_internal(md->key) &&
                !GRPC_LOG_IF_ERROR(
                    "validate_metadata",
                    grpc_validate_header_nonbin_value_is_legal(md->value))) {
@@ -1225,7 +1231,7 @@ static void post_batch_completion(batch_control* bctl) {
 }
 
 static void finish_batch_step(batch_control* bctl) {
-  if (GPR_UNLIKELY(gpr_unref(&bctl->steps_to_complete))) {
+  if (GPR_UNLIKELY(bctl->completed_batch_step())) {
     post_batch_completion(bctl);
   }
 }
@@ -1866,7 +1872,7 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
   if (!is_notify_tag_closure) {
     GPR_ASSERT(grpc_cq_begin_op(call->cq, notify_tag));
   }
-  gpr_ref_init(&bctl->steps_to_complete, (has_send_ops ? 1 : 0) + num_recv_ops);
+  bctl->set_num_steps_to_complete((has_send_ops ? 1 : 0) + num_recv_ops);
 
   if (has_send_ops) {
     GRPC_CLOSURE_INIT(&bctl->finish_batch, finish_batch, bctl,