grpc 1.41.0 → 1.42.0.pre1

data/src/core/lib/slice/slice_refcount.h

@@ -0,0 +1,121 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_SLICE_SLICE_REFCOUNT_H
+#define GRPC_CORE_LIB_SLICE_SLICE_REFCOUNT_H
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/slice/slice_refcount_base.h"
+#include "src/core/lib/slice/static_slice.h"
+
+// Interned slices have specific fast-path operations for hashing. To inline
+// these operations, we need to forward declare them here.
+extern uint32_t grpc_static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
+
+namespace grpc_core {
+
+extern grpc_slice_refcount kNoopRefcount;
+
+struct InternedSliceRefcount {
+  static void Destroy(void* arg) {
+    auto* rc = static_cast<InternedSliceRefcount*>(arg);
+    rc->~InternedSliceRefcount();
+    gpr_free(rc);
+  }
+
+  InternedSliceRefcount(size_t length, uint32_t hash,
+                        InternedSliceRefcount* bucket_next)
+      : base(grpc_slice_refcount::Type::INTERNED, &refcnt, Destroy, this, &sub),
+        sub(grpc_slice_refcount::Type::REGULAR, &refcnt, Destroy, this, &sub),
+        length(length),
+        hash(hash),
+        bucket_next(bucket_next) {}
+
+  ~InternedSliceRefcount();
+
+  grpc_slice_refcount base;
+  grpc_slice_refcount sub;
+  const size_t length;
+  RefCount refcnt;
+  const uint32_t hash;
+  InternedSliceRefcount* bucket_next;
+};
+
+}  // namespace grpc_core
+
+inline size_t grpc_refcounted_slice_length(const grpc_slice& slice) {
+  GPR_DEBUG_ASSERT(slice.refcount != nullptr);
+  return slice.data.refcounted.length;
+}
+
+inline const uint8_t* grpc_refcounted_slice_data(const grpc_slice& slice) {
+  GPR_DEBUG_ASSERT(slice.refcount != nullptr);
+  return slice.data.refcounted.bytes;
+}
+
+inline int grpc_slice_refcount::Eq(const grpc_slice& a, const grpc_slice& b) {
+  GPR_DEBUG_ASSERT(a.refcount != nullptr);
+  GPR_DEBUG_ASSERT(a.refcount == this);
+  switch (ref_type_) {
+    case Type::STATIC:
+      GPR_DEBUG_ASSERT(
+          (GRPC_STATIC_METADATA_INDEX(a) == GRPC_STATIC_METADATA_INDEX(b)) ==
+          (a.refcount == b.refcount));
+      ABSL_FALLTHROUGH_INTENDED;
+    case Type::INTERNED:
+      return a.refcount == b.refcount;
+    case Type::NOP:
+    case Type::REGULAR:
+      break;
+  }
+  if (grpc_refcounted_slice_length(a) != GRPC_SLICE_LENGTH(b)) return false;
+  if (grpc_refcounted_slice_length(a) == 0) return true;
+  return 0 == memcmp(grpc_refcounted_slice_data(a), GRPC_SLICE_START_PTR(b),
+                     grpc_refcounted_slice_length(a));
+}
+
+inline uint32_t grpc_slice_refcount::Hash(const grpc_slice& slice) {
+  GPR_DEBUG_ASSERT(slice.refcount != nullptr);
+  GPR_DEBUG_ASSERT(slice.refcount == this);
+  switch (ref_type_) {
+    case Type::STATIC:
+      return ::grpc_static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(
+          slice)];
+    case Type::INTERNED:
+      return reinterpret_cast<grpc_core::InternedSliceRefcount*>(slice.refcount)
+          ->hash;
+    case Type::NOP:
+    case Type::REGULAR:
+      break;
+  }
+  return gpr_murmur_hash3(grpc_refcounted_slice_data(slice),
+                          grpc_refcounted_slice_length(slice),
+                          grpc_core::g_hash_seed);
+}
+
+inline const grpc_slice& grpc_slice_ref_internal(const grpc_slice& slice) {
+  if (slice.refcount) {
+    slice.refcount->Ref();
+  }
+  return slice;
+}
+
+inline void grpc_slice_unref_internal(const grpc_slice& slice) {
+  if (slice.refcount) {
+    slice.refcount->Unref();
+  }
+}
+
+#endif /* GRPC_CORE_LIB_SLICE_SLICE_REFCOUNT_H */

data/src/core/lib/slice/slice_refcount_base.h

@@ -0,0 +1,173 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_SLICE_SLICE_REFCOUNT_BASE_H
+#define GRPC_CORE_LIB_SLICE_SLICE_REFCOUNT_BASE_H
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/slice.h>
+
+#include "src/core/lib/gprpp/ref_counted.h"
+
+// grpc_slice_refcount : A reference count for grpc_slice.
+//
+// Non-inlined grpc_slice objects are refcounted. Historically this was
+// implemented via grpc_slice_refcount, a C-style polymorphic class using a
+// manually managed vtable of operations. Subclasses would define their own
+// vtable; the 'virtual' methods (ref, unref, equals and hash) would simply call
+// the function pointers in the vtable as necessary.
+//
+// Unfortunately, this leads to some inefficiencies in the generated code that
+// can be improved upon. For example, equality checking for interned slices is a
+// simple equality check on the refcount pointer. With the vtable approach, this
+// would translate to roughly the following (high-level) instructions:
+//
+// grpc_slice_equals(slice1, slice2):
+//   load vtable->eq -> eq_func
+//   call eq_func(slice1, slice2)
+//
+// interned_slice_equals(slice1, slice2)
+//   load slice1.ref -> r1
+//   load slice2.ref -> r2
+//   cmp r1, r2 -> retval
+//   ret retval
+//
+// This leads to a function call for a function defined in another translation
+// unit, which imposes memory barriers, which reduces the compiler's ability to
+// optimize (in addition to the added overhead of call/ret). Additionally, it
+// may be harder to reason about branch prediction when we're jumping to
+// essentially arbitrarily provided function pointers.
+//
+// In addition, it is arguable that while virtualization was helpful for
+// Equals()/Hash() methods, that it was fundamentally unnecessary for
+// Ref()/Unref().
+//
+// Instead, grpc_slice_refcount provides the same functionality as the C-style
+// virtual class, but in a de-virtualized manner - Eq(), Hash(), Ref() and
+// Unref() are provided within this header file. Fastpaths for Eq()/Hash()
+// (interned and static metadata slices), as well as the Ref() operation, can
+// all be inlined without any memory barriers.
+//
+// It does this by:
+// 1. Using grpc_core::RefCount<> (header-only) for Ref/Unref. Two special cases
+//    need support: No-op ref/unref (eg. static metadata slices) and stream
+//    slice references (where all the slices share the streamref). This is in
+//    addition to the normal case of '1 slice, 1 ref'.
+//    To support these cases, we explicitly track a nullable pointer to the
+//    underlying RefCount<>. No-op ref/unref is used by checking the pointer for
+//    null, and doing nothing if it is. Both stream slice refs and 'normal'
+//    slices use the same path for Ref/Unref (by targeting the non-null
+//    pointer).
+//
+// 2. introducing the notion of grpc_slice_refcount::Type. This describes if a
+//    slice ref is used by a static metadata slice, an interned slice, or other
+//    slices. We switch on the slice ref type in order to provide fastpaths for
+//    Equals() and Hash().
+//
+// In total, this saves us roughly 1-2% latency for unary calls, with smaller
+// calls benefitting. The effect is present, but not as useful, for larger calls
+// where the cost of sending the data dominates.
+// TODO(arjunroy): Investigate if this can be removed with strongly typed
+// grpc_slices.
+struct grpc_slice_refcount {
+ public:
+  enum class Type {
+    STATIC,    // Refcount for a static metadata slice.
+    INTERNED,  // Refcount for an interned slice.
+    NOP,       // No-Op
+    REGULAR    // Refcount for non-static-metadata, non-interned slices.
+  };
+  typedef void (*DestroyerFn)(void*);
+
+  grpc_slice_refcount() = default;
+
+  explicit grpc_slice_refcount(Type t) : ref_type_(t) {}
+
+  explicit grpc_slice_refcount(grpc_slice_refcount* sub) : sub_refcount_(sub) {}
+  // Regular constructor for grpc_slice_refcount.
+  //
+  // Parameters:
+  //  1. grpc_slice_refcount::Type type
+  //  Whether we are the refcount for a static
+  //  metadata slice, an interned slice, or any other kind of slice.
+  //
+  //  2. RefCount* ref
+  //  The pointer to the actual underlying grpc_core::RefCount. Rather than
+  //  performing struct offset computations as in the original implementation to
+  //  get to the refcount, which requires a virtual method, we devirtualize by
+  //  using a nullable pointer to allow a single pair of Ref/Unref methods.
+  //
+  //  3. DestroyerFn destroyer_fn
+  //  Called when the refcount goes to 0, with destroyer_arg as parameter.
+  //
+  //  4. void* destroyer_arg
+  //  Argument for the virtualized destructor.
+  //
+  //  5. grpc_slice_refcount* sub
+  //  Argument used for interned slices.
+  grpc_slice_refcount(grpc_slice_refcount::Type type, grpc_core::RefCount* ref,
+                      DestroyerFn destroyer_fn, void* destroyer_arg,
+                      grpc_slice_refcount* sub)
+      : ref_(ref),
+        ref_type_(type),
+        sub_refcount_(sub),
+        dest_fn_(destroyer_fn),
+        destroy_fn_arg_(destroyer_arg) {}
+  // Initializer for static refcounts.
+  grpc_slice_refcount(grpc_slice_refcount* sub, Type type)
+      : ref_type_(type), sub_refcount_(sub) {}
+
+  Type GetType() const { return ref_type_; }
+
+  int Eq(const grpc_slice& a, const grpc_slice& b);
+
+  uint32_t Hash(const grpc_slice& slice);
+  void Ref() {
+    if (ref_ == nullptr) return;
+    ref_->RefNonZero();
+  }
+  void Unref() {
+    if (ref_ == nullptr) return;
+    if (ref_->Unref()) {
+      dest_fn_(destroy_fn_arg_);
+    }
+  }
+
+  grpc_slice_refcount* sub_refcount() const { return sub_refcount_; }
+
+ private:
+  grpc_core::RefCount* ref_ = nullptr;
+  const Type ref_type_ = Type::REGULAR;
+  grpc_slice_refcount* sub_refcount_ = this;
+  DestroyerFn dest_fn_ = nullptr;
+  void* destroy_fn_arg_ = nullptr;
+};
+
+namespace grpc_core {
+
+struct StaticSliceRefcount {
+  static grpc_slice_refcount kStaticSubRefcount;
+
+  explicit StaticSliceRefcount(uint32_t index)
+      : base(&kStaticSubRefcount, grpc_slice_refcount::Type::STATIC),
+        index(index) {}
+
+  grpc_slice_refcount base;
+  const uint32_t index;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_LIB_SLICE_SLICE_REFCOUNT_BASE_H

data/src/core/lib/slice/slice_split.cc

@@ -0,0 +1,100 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/slice/slice_split.h"
+
+#include <string.h>
+
+#include <grpc/support/log.h>
+
+/** Finds the initial (\a begin) and final (\a end) offsets of the next
+ * substring from \a str + \a read_offset until the next \a sep or the end of \a
+ * str.
+ *
+ * Returns 1 and updates \a begin and \a end. Returns 0 otherwise. */
+static int slice_find_separator_offset(const grpc_slice str, const char* sep,
+                                       const size_t read_offset, size_t* begin,
+                                       size_t* end) {
+  size_t i;
+  const uint8_t* str_ptr = GRPC_SLICE_START_PTR(str) + read_offset;
+  const size_t str_len = GRPC_SLICE_LENGTH(str) - read_offset;
+  const size_t sep_len = strlen(sep);
+  if (str_len < sep_len) {
+    return 0;
+  }
+
+  for (i = 0; i <= str_len - sep_len; i++) {
+    if (memcmp(str_ptr + i, sep, sep_len) == 0) {
+      *begin = read_offset;
+      *end = read_offset + i;
+      return 1;
+    }
+  }
+  return 0;
+}
+
+static void skip_leading_trailing_spaces(const uint8_t* str_buffer,
+                                         size_t* begin, size_t* end) {
+  while (*begin < *end && str_buffer[*begin] == ' ') {
+    (*begin)++;
+  }
+  while (*begin < *end && str_buffer[*end - 1] == ' ') {
+    (*end)--;
+  }
+}
+
+static void grpc_slice_split_inner(grpc_slice str, const char* sep,
+                                   grpc_slice_buffer* dst, bool no_space) {
+  const size_t sep_len = strlen(sep);
+  size_t begin, end;
+  const uint8_t* str_buffer = GRPC_SLICE_START_PTR(str);
+  size_t sep_pos;
+
+  GPR_ASSERT(sep_len > 0);
+
+  if (slice_find_separator_offset(str, sep, 0, &begin, &end) != 0) {
+    do {
+      sep_pos = end;
+      if (no_space) {
+        skip_leading_trailing_spaces(str_buffer, &begin, &end);
+      }
+      grpc_slice_buffer_add_indexed(dst, grpc_slice_sub(str, begin, end));
+    } while (slice_find_separator_offset(str, sep, sep_pos + sep_len, &begin,
+                                         &end) != 0);
+    begin = sep_pos + sep_len;
+    end = GRPC_SLICE_LENGTH(str);
+    if (no_space) {
+      skip_leading_trailing_spaces(str_buffer, &begin, &end);
+    }
+    grpc_slice_buffer_add_indexed(dst, grpc_slice_sub(str, begin, end));
+  } else { /* no sep found, add whole input */
+    begin = 0;
+    end = GRPC_SLICE_LENGTH(str);
+    if (no_space) {
+      skip_leading_trailing_spaces(str_buffer, &begin, &end);
+    }
+    grpc_slice_buffer_add_indexed(dst, grpc_slice_sub(str, begin, end));
+  }
+}
+
+void grpc_slice_split(grpc_slice str, const char* sep, grpc_slice_buffer* dst) {
+  grpc_slice_split_inner(str, sep, dst, false);
+}
+
+void grpc_slice_split_without_space(grpc_slice str, const char* sep,
+                                    grpc_slice_buffer* dst) {
+  grpc_slice_split_inner(str, sep, dst, true);
+}

data/src/core/lib/slice/slice_split.h

@@ -0,0 +1,40 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SLICE_SLICE_SPLIT_H
+#define GRPC_CORE_LIB_SLICE_SLICE_SPLIT_H
+
+#include <grpc/support/port_platform.h>
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include <grpc/slice.h>
+#include <grpc/slice_buffer.h>
+
+/** Split \a str by the separator \a sep. Results are stored in \a dst, which
+ * should be a properly initialized instance. */
+void grpc_slice_split(grpc_slice str, const char* sep, grpc_slice_buffer* dst);
+
+/** Split \a str by the separator \a sep and remove the leading and trailing
+ * spaces of each resulting token. Results are stored in \a dst, which should be
+ * a properly initialized instance. */
+void grpc_slice_split_without_space(grpc_slice str, const char* sep,
+                                    grpc_slice_buffer* dst);
+
+#endif /* GRPC_CORE_LIB_SLICE_SLICE_SPLIT_H */

data/src/core/lib/slice/slice_string_helpers.cc

@@ -20,10 +20,6 @@
 
 #include "src/core/lib/slice/slice_string_helpers.h"
 
-#include <string.h>
-
-#include <grpc/support/log.h>
-
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/memory.h"
 #include "src/core/lib/slice/slice_internal.h"
@@ -41,85 +37,6 @@ grpc_slice grpc_dump_slice_to_slice(const grpc_slice& s, uint32_t flags) {
   return grpc_slice_from_moved_buffer(std::move(ptr), len);
 }
 
-/** Finds the initial (\a begin) and final (\a end) offsets of the next
- * substring from \a str + \a read_offset until the next \a sep or the end of \a
- * str.
- *
- * Returns 1 and updates \a begin and \a end. Returns 0 otherwise. */
-static int slice_find_separator_offset(const grpc_slice str, const char* sep,
-                                       const size_t read_offset, size_t* begin,
-                                       size_t* end) {
-  size_t i;
-  const uint8_t* str_ptr = GRPC_SLICE_START_PTR(str) + read_offset;
-  const size_t str_len = GRPC_SLICE_LENGTH(str) - read_offset;
-  const size_t sep_len = strlen(sep);
-  if (str_len < sep_len) {
-    return 0;
-  }
-
-  for (i = 0; i <= str_len - sep_len; i++) {
-    if (memcmp(str_ptr + i, sep, sep_len) == 0) {
-      *begin = read_offset;
-      *end = read_offset + i;
-      return 1;
-    }
-  }
-  return 0;
-}
-
-static void skip_leading_trailing_spaces(const uint8_t* str_buffer,
-                                         size_t* begin, size_t* end) {
-  while (*begin < *end && str_buffer[*begin] == ' ') {
-    (*begin)++;
-  }
-  while (*begin < *end && str_buffer[*end - 1] == ' ') {
-    (*end)--;
-  }
-}
-
-static void grpc_slice_split_inner(grpc_slice str, const char* sep,
-                                   grpc_slice_buffer* dst, bool no_space) {
-  const size_t sep_len = strlen(sep);
-  size_t begin, end;
-  const uint8_t* str_buffer = GRPC_SLICE_START_PTR(str);
-  size_t sep_pos;
-
-  GPR_ASSERT(sep_len > 0);
-
-  if (slice_find_separator_offset(str, sep, 0, &begin, &end) != 0) {
-    do {
-      sep_pos = end;
-      if (no_space) {
-        skip_leading_trailing_spaces(str_buffer, &begin, &end);
-      }
-      grpc_slice_buffer_add_indexed(dst, grpc_slice_sub(str, begin, end));
-    } while (slice_find_separator_offset(str, sep, sep_pos + sep_len, &begin,
-                                         &end) != 0);
-    begin = sep_pos + sep_len;
-    end = GRPC_SLICE_LENGTH(str);
-    if (no_space) {
-      skip_leading_trailing_spaces(str_buffer, &begin, &end);
-    }
-    grpc_slice_buffer_add_indexed(dst, grpc_slice_sub(str, begin, end));
-  } else { /* no sep found, add whole input */
-    begin = 0;
-    end = GRPC_SLICE_LENGTH(str);
-    if (no_space) {
-      skip_leading_trailing_spaces(str_buffer, &begin, &end);
-    }
-    grpc_slice_buffer_add_indexed(dst, grpc_slice_sub(str, begin, end));
-  }
-}
-
-void grpc_slice_split(grpc_slice str, const char* sep, grpc_slice_buffer* dst) {
-  grpc_slice_split_inner(str, sep, dst, false);
-}
-
-void grpc_slice_split_without_space(grpc_slice str, const char* sep,
-                                    grpc_slice_buffer* dst) {
-  grpc_slice_split_inner(str, sep, dst, true);
-}
-
 bool grpc_parse_slice_to_uint32(grpc_slice str, uint32_t* result) {
   return gpr_parse_bytes_to_uint32(
              reinterpret_cast<const char*> GRPC_SLICE_START_PTR(str),

data/src/core/lib/slice/slice_string_helpers.h

@@ -25,7 +25,6 @@
 #include <stddef.h>
 
 #include <grpc/slice.h>
-#include <grpc/slice_buffer.h>
 
 #include "src/core/lib/gpr/string.h"
 
@@ -34,16 +33,6 @@ char* grpc_dump_slice(const grpc_slice& slice, uint32_t flags);
 /* Calls gpr_dump on a slice and returns the result as a slice. */
 grpc_slice grpc_dump_slice_to_slice(const grpc_slice& slice, uint32_t flags);
 
-/** Split \a str by the separator \a sep. Results are stored in \a dst, which
- * should be a properly initialized instance. */
-void grpc_slice_split(grpc_slice str, const char* sep, grpc_slice_buffer* dst);
-
-/** Split \a str by the separator \a sep and remove the leading and trailing
- * spaces of each resulting token. Results are stored in \a dst, which should be
- * a properly initialized instance. */
-void grpc_slice_split_without_space(grpc_slice str, const char* sep,
-                                    grpc_slice_buffer* dst);
-
 bool grpc_parse_slice_to_uint32(grpc_slice str, uint32_t* result);
 
 #endif /* GRPC_CORE_LIB_SLICE_SLICE_STRING_HELPERS_H */