grpc 1.30.2

data/src/core/lib/slice/slice_intern.cc

@@ -0,0 +1,375 @@
+/*
+ *
+ * Copyright 2016 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/slice/slice_utils.h"
+
+#include <inttypes.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/lib/gpr/murmur_hash.h"
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/iomgr/iomgr_internal.h" /* for iomgr_abort_on_leaks() */
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/slice/slice_string_helpers.h"
+#include "src/core/lib/transport/static_metadata.h"
+
+#define LOG2_SHARD_COUNT 5
+#define SHARD_COUNT (1 << LOG2_SHARD_COUNT)
+#define INITIAL_SHARD_CAPACITY 8
+
+#define TABLE_IDX(hash, capacity) (((hash) >> LOG2_SHARD_COUNT) % (capacity))
+#define SHARD_IDX(hash) ((hash) & ((1 << LOG2_SHARD_COUNT) - 1))
+
+using grpc_core::InternedSliceRefcount;
+
+typedef struct slice_shard {
+  gpr_mu mu;
+  InternedSliceRefcount** strs;
+  size_t count;
+  size_t capacity;
+} slice_shard;
+
+static slice_shard g_shards[SHARD_COUNT];
+
+struct static_metadata_hash_ent {
+  uint32_t hash;
+  uint32_t idx;
+};
+static static_metadata_hash_ent
+    static_metadata_hash[4 * GRPC_STATIC_MDSTR_COUNT];
+static uint32_t max_static_metadata_hash_probe;
+uint32_t grpc_static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
+
+namespace grpc_core {
+
+/* hash seed: decided at initialization time */
+uint32_t g_hash_seed;
+static bool g_forced_hash_seed = false;
+
+InternedSliceRefcount::~InternedSliceRefcount() {
+  slice_shard* shard = &g_shards[SHARD_IDX(this->hash)];
+  MutexLock lock(&shard->mu);
+  InternedSliceRefcount** prev_next;
+  InternedSliceRefcount* cur;
+  for (prev_next = &shard->strs[TABLE_IDX(this->hash, shard->capacity)],
+      cur = *prev_next;
+       cur != this; prev_next = &cur->bucket_next, cur = cur->bucket_next)
+    ;
+  *prev_next = cur->bucket_next;
+  shard->count--;
+}
+
+}  // namespace grpc_core
+
+static void grow_shard(slice_shard* shard) {
+  GPR_TIMER_SCOPE("grow_strtab", 0);
+
+  size_t capacity = shard->capacity * 2;
+  size_t i;
+  InternedSliceRefcount** strtab;
+  InternedSliceRefcount *s, *next;
+
+  strtab = static_cast<InternedSliceRefcount**>(
+      gpr_zalloc(sizeof(InternedSliceRefcount*) * capacity));
+
+  for (i = 0; i < shard->capacity; i++) {
+    for (s = shard->strs[i]; s; s = next) {
+      size_t idx = TABLE_IDX(s->hash, capacity);
+      next = s->bucket_next;
+      s->bucket_next = strtab[idx];
+      strtab[idx] = s;
+    }
+  }
+  gpr_free(shard->strs);
+  shard->strs = strtab;
+  shard->capacity = capacity;
+}
+
+grpc_core::InternedSlice::InternedSlice(InternedSliceRefcount* s) {
+  refcount = &s->base;
+  data.refcounted.bytes = reinterpret_cast<uint8_t*>(s + 1);
+  data.refcounted.length = s->length;
+}
+
+uint32_t grpc_slice_default_hash_impl(grpc_slice s) {
+  return gpr_murmur_hash3(GRPC_SLICE_START_PTR(s), GRPC_SLICE_LENGTH(s),
+                          grpc_core::g_hash_seed);
+}
+
+uint32_t grpc_static_slice_hash(grpc_slice s) {
+  return grpc_static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(s)];
+}
+
+int grpc_static_slice_eq(grpc_slice a, grpc_slice b) {
+  return GRPC_STATIC_METADATA_INDEX(a) == GRPC_STATIC_METADATA_INDEX(b);
+}
+
+uint32_t grpc_slice_hash(grpc_slice s) { return grpc_slice_hash_internal(s); }
+
+grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
+                                          bool* returned_slice_is_different) {
+  if (GRPC_IS_STATIC_METADATA_STRING(slice)) {
+    return slice;
+  }
+
+  uint32_t hash = grpc_slice_hash_internal(slice);
+  for (uint32_t i = 0; i <= max_static_metadata_hash_probe; i++) {
+    static_metadata_hash_ent ent =
+        static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)];
+    const grpc_core::StaticMetadataSlice* static_slice_table =
+        grpc_static_slice_table();
+    if (ent.hash == hash && ent.idx < GRPC_STATIC_MDSTR_COUNT &&
+        grpc_slice_eq_static_interned(slice, static_slice_table[ent.idx])) {
+      *returned_slice_is_different = true;
+      return static_slice_table[ent.idx];
+    }
+  }
+
+  return slice;
+}
+
+grpc_slice grpc_slice_intern(grpc_slice slice) {
+  /* TODO(arjunroy): At present, this is capable of returning either a static or
+     an interned slice. This yields weirdness like the constructor for
+     ManagedMemorySlice instantiating itself as an instance of a derived type
+     (StaticMetadataSlice or InternedSlice). Should reexamine. */
+  return grpc_core::ManagedMemorySlice(&slice);
+}
+
+// Attempt to see if the provided slice or string matches a static slice.
+// SliceArgs is either a const grpc_slice& or const pair<const char*, size_t>&.
+// In either case, hash is the pre-computed hash value.
+//
+// Returns: a matching static slice, or null.
+template <typename SliceArgs>
+static const grpc_core::StaticMetadataSlice* MatchStaticSlice(
+    uint32_t hash, const SliceArgs& args) {
+  for (uint32_t i = 0; i <= max_static_metadata_hash_probe; i++) {
+    static_metadata_hash_ent ent =
+        static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)];
+    const grpc_core::StaticMetadataSlice* static_slice_table =
+        grpc_static_slice_table();
+    if (ent.hash == hash && ent.idx < GRPC_STATIC_MDSTR_COUNT &&
+        static_slice_table[ent.idx] == args) {
+      return &static_slice_table[ent.idx];
+    }
+  }
+  return nullptr;
+}
+
+// Helper methods to enable us to select appropriately overloaded slice methods
+// whether we're dealing with a slice, or a buffer with length, when interning
+// strings. Helpers for FindOrCreateInternedSlice().
+static const char* GetBuffer(const std::pair<const char*, size_t>& buflen) {
+  return buflen.first;
+}
+static size_t GetLength(const std::pair<const char*, size_t>& buflen) {
+  return buflen.second;
+}
+static const void* GetBuffer(const grpc_slice& slice) {
+  return GRPC_SLICE_START_PTR(slice);
+}
+static size_t GetLength(const grpc_slice& slice) {
+  return GRPC_SLICE_LENGTH(slice);
+}
+
+// Creates an interned slice for a string that does not currently exist in the
+// intern table. SliceArgs is either a const grpc_slice& or a const
+// pair<const char*, size_t>&. Hash is the pre-computed hash value. We must
+// already hold the shard lock. Helper for FindOrCreateInternedSlice().
+//
+// Returns: a newly interned slice.
+template <typename SliceArgs>
+static InternedSliceRefcount* InternNewStringLocked(slice_shard* shard,
+                                                    size_t shard_idx,
+                                                    uint32_t hash,
+                                                    const SliceArgs& args) {
+  /* string data goes after the internal_string header */
+  size_t len = GetLength(args);
+  const void* buffer = GetBuffer(args);
+  InternedSliceRefcount* s =
+      static_cast<InternedSliceRefcount*>(gpr_malloc(sizeof(*s) + len));
+  new (s) grpc_core::InternedSliceRefcount(len, hash, shard->strs[shard_idx]);
+  // TODO(arjunroy): Investigate why hpack tried to intern the nullptr string.
+  // https://github.com/grpc/grpc/pull/20110#issuecomment-526729282
+  if (len > 0) {
+    memcpy(reinterpret_cast<char*>(s + 1), buffer, len);
+  }
+  shard->strs[shard_idx] = s;
+  shard->count++;
+  if (shard->count > shard->capacity * 2) {
+    grow_shard(shard);
+  }
+  return s;
+}
+
+// Attempt to see if the provided slice or string matches an existing interned
+// slice. SliceArgs... is either a const grpc_slice& or a string and length. In
+// either case, hash is the pre-computed hash value.  We must already hold the
+// shard lock. Helper for FindOrCreateInternedSlice().
+//
+// Returns: a pre-existing matching static slice, or null.
+template <typename SliceArgs>
+static InternedSliceRefcount* MatchInternedSliceLocked(uint32_t hash,
+                                                       size_t idx,
+                                                       const SliceArgs& args) {
+  InternedSliceRefcount* s;
+  slice_shard* shard = &g_shards[SHARD_IDX(hash)];
+  /* search for an existing string */
+  for (s = shard->strs[idx]; s; s = s->bucket_next) {
+    if (s->hash == hash && grpc_core::InternedSlice(s) == args) {
+      if (s->refcnt.RefIfNonZero()) {
+        return s;
+      }
+    }
+  }
+  return nullptr;
+}
+
+// Attempt to see if the provided slice or string matches an existing interned
+// slice, and failing that, create an interned slice with its contents. Returns
+// either the existing matching interned slice or the newly created one.
+// SliceArgs is either a const grpc_slice& or const pair<const char*, size_t>&.
+// In either case, hash is the pre-computed hash value. We do not hold the
+// shard lock here, but do take it.
+//
+// Returns: an interned slice, either pre-existing/matched or newly created.
+template <typename SliceArgs>
+static InternedSliceRefcount* FindOrCreateInternedSlice(uint32_t hash,
+                                                        const SliceArgs& args) {
+  slice_shard* shard = &g_shards[SHARD_IDX(hash)];
+  gpr_mu_lock(&shard->mu);
+  const size_t idx = TABLE_IDX(hash, shard->capacity);
+  InternedSliceRefcount* s = MatchInternedSliceLocked(hash, idx, args);
+  if (s == nullptr) {
+    s = InternNewStringLocked(shard, idx, hash, args);
+  }
+  gpr_mu_unlock(&shard->mu);
+  return s;
+}
+
+grpc_core::ManagedMemorySlice::ManagedMemorySlice(const char* string)
+    : grpc_core::ManagedMemorySlice::ManagedMemorySlice(string,
+                                                        strlen(string)) {}
+
+grpc_core::ManagedMemorySlice::ManagedMemorySlice(const char* string,
+                                                  size_t len) {
+  GPR_TIMER_SCOPE("grpc_slice_intern", 0);
+  const uint32_t hash = gpr_murmur_hash3(string, len, g_hash_seed);
+  const StaticMetadataSlice* static_slice =
+      MatchStaticSlice(hash, std::pair<const char*, size_t>(string, len));
+  if (static_slice) {
+    *this = *static_slice;
+  } else {
+    *this = grpc_core::InternedSlice(FindOrCreateInternedSlice(
+        hash, std::pair<const char*, size_t>(string, len)));
+  }
+}
+
+grpc_core::ManagedMemorySlice::ManagedMemorySlice(const grpc_slice* slice_ptr) {
+  GPR_TIMER_SCOPE("grpc_slice_intern", 0);
+  const grpc_slice& slice = *slice_ptr;
+  if (GRPC_IS_STATIC_METADATA_STRING(slice)) {
+    *this = static_cast<const grpc_core::StaticMetadataSlice&>(slice);
+    return;
+  }
+  const uint32_t hash = grpc_slice_hash_internal(slice);
+  const StaticMetadataSlice* static_slice = MatchStaticSlice(hash, slice);
+  if (static_slice) {
+    *this = *static_slice;
+  } else {
+    *this = grpc_core::InternedSlice(FindOrCreateInternedSlice(hash, slice));
+  }
+}
+
+void grpc_test_only_set_slice_hash_seed(uint32_t seed) {
+  grpc_core::g_hash_seed = seed;
+  grpc_core::g_forced_hash_seed = true;
+}
+
+void grpc_slice_intern_init(void) {
+  if (!grpc_core::g_forced_hash_seed) {
+    grpc_core::g_hash_seed =
+        static_cast<uint32_t>(gpr_now(GPR_CLOCK_REALTIME).tv_nsec);
+  }
+  for (size_t i = 0; i < SHARD_COUNT; i++) {
+    slice_shard* shard = &g_shards[i];
+    gpr_mu_init(&shard->mu);
+    shard->count = 0;
+    shard->capacity = INITIAL_SHARD_CAPACITY;
+    shard->strs = static_cast<InternedSliceRefcount**>(
+        gpr_zalloc(sizeof(*shard->strs) * shard->capacity));
+  }
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(static_metadata_hash); i++) {
+    static_metadata_hash[i].hash = 0;
+    static_metadata_hash[i].idx = GRPC_STATIC_MDSTR_COUNT;
+  }
+  max_static_metadata_hash_probe = 0;
+  const grpc_core::StaticMetadataSlice* static_slice_table =
+      grpc_static_slice_table();
+  for (size_t i = 0; i < GRPC_STATIC_MDSTR_COUNT; i++) {
+    grpc_static_metadata_hash_values[i] =
+        grpc_slice_default_hash_internal(static_slice_table[i]);
+    for (size_t j = 0; j < GPR_ARRAY_SIZE(static_metadata_hash); j++) {
+      size_t slot = (grpc_static_metadata_hash_values[i] + j) %
+                    GPR_ARRAY_SIZE(static_metadata_hash);
+      if (static_metadata_hash[slot].idx == GRPC_STATIC_MDSTR_COUNT) {
+        static_metadata_hash[slot].hash = grpc_static_metadata_hash_values[i];
+        static_metadata_hash[slot].idx = static_cast<uint32_t>(i);
+        if (j > max_static_metadata_hash_probe) {
+          max_static_metadata_hash_probe = static_cast<uint32_t>(j);
+        }
+        break;
+      }
+    }
+  }
+  // Handle KV hash for all static mdelems.
+  for (size_t i = 0; i < GRPC_STATIC_MDELEM_COUNT; ++i) {
+    grpc_static_mdelem_table()[i].HashInit();
+  }
+}
+
+void grpc_slice_intern_shutdown(void) {
+  for (size_t i = 0; i < SHARD_COUNT; i++) {
+    slice_shard* shard = &g_shards[i];
+    gpr_mu_destroy(&shard->mu);
+    /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */
+    if (shard->count != 0) {
+      gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata strings were leaked",
+              shard->count);
+      for (size_t j = 0; j < shard->capacity; j++) {
+        for (InternedSliceRefcount* s = shard->strs[j]; s; s = s->bucket_next) {
+          char* text = grpc_dump_slice(grpc_core::InternedSlice(s),
+                                       GPR_DUMP_HEX | GPR_DUMP_ASCII);
+          gpr_log(GPR_DEBUG, "LEAKED: %s", text);
+          gpr_free(text);
+        }
+      }
+      if (grpc_iomgr_abort_on_leaks()) {
+        abort();
+      }
+    }
+    gpr_free(shard->strs);
+  }
+}

data/src/core/lib/slice/slice_internal.h

@@ -0,0 +1,364 @@
+/*
+ *
+ * Copyright 2016 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H
+#define GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/support/log.h>
+
+#include <grpc/slice.h>
+#include <grpc/slice_buffer.h>
+#include <string.h>
+
+#include "src/core/lib/gpr/murmur_hash.h"
+#include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/slice/slice_utils.h"
+#include "src/core/lib/transport/static_metadata.h"
+
+// Interned slices have specific fast-path operations for hashing. To inline
+// these operations, we need to forward declare them here.
+extern uint32_t grpc_static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
+
+// grpc_slice_refcount : A reference count for grpc_slice.
+//
+// Non-inlined grpc_slice objects are refcounted. Historically this was
+// implemented via grpc_slice_refcount, a C-style polymorphic class using a
+// manually managed vtable of operations. Subclasses would define their own
+// vtable; the 'virtual' methods (ref, unref, equals and hash) would simply call
+// the function pointers in the vtable as necessary.
+//
+// Unfortunately, this leads to some inefficiencies in the generated code that
+// can be improved upon. For example, equality checking for interned slices is a
+// simple equality check on the refcount pointer. With the vtable approach, this
+// would translate to roughly the following (high-level) instructions:
+//
+// grpc_slice_equals(slice1, slice2):
+//   load vtable->eq -> eq_func
+//   call eq_func(slice1, slice2)
+//
+// interned_slice_equals(slice1, slice2)
+//   load slice1.ref -> r1
+//   load slice2.ref -> r2
+//   cmp r1, r2 -> retval
+//   ret retval
+//
+// This leads to a function call for a function defined in another translation
+// unit, which imposes memory barriers, which reduces the compiler's ability to
+// optimize (in addition to the added overhead of call/ret). Additionally, it
+// may be harder to reason about branch prediction when we're jumping to
+// essentially arbitrarily provided function pointers.
+//
+// In addition, it is arguable that while virtualization was helpful for
+// Equals()/Hash() methods, that it was fundamentally unnecessary for
+// Ref()/Unref().
+//
+// Instead, grpc_slice_refcount provides the same functionality as the C-style
+// virtual class, but in a de-virtualized manner - Eq(), Hash(), Ref() and
+// Unref() are provided within this header file. Fastpaths for Eq()/Hash()
+// (interned and static metadata slices), as well as the Ref() operation, can
+// all be inlined without any memory barriers.
+//
+// It does this by:
+// 1. Using grpc_core::RefCount<> (header-only) for Ref/Unref. Two special cases
+//    need support: No-op ref/unref (eg. static metadata slices) and stream
+//    slice references (where all the slices share the streamref). This is in
+//    addition to the normal case of '1 slice, 1 ref'.
+//    To support these cases, we explicitly track a nullable pointer to the
+//    underlying RefCount<>. No-op ref/unref is used by checking the pointer for
+//    null, and doing nothing if it is. Both stream slice refs and 'normal'
+//    slices use the same path for Ref/Unref (by targeting the non-null
+//    pointer).
+//
+// 2. introducing the notion of grpc_slice_refcount::Type. This describes if a
+//    slice ref is used by a static metadata slice, an interned slice, or other
+//    slices. We switch on the slice ref type in order to provide fastpaths for
+//    Equals() and Hash().
+//
+// In total, this saves us roughly 1-2% latency for unary calls, with smaller
+// calls benefitting. The effect is present, but not as useful, for larger calls
+// where the cost of sending the data dominates.
+// TODO(arjunroy): Investigate if this can be removed with strongly typed
+// grpc_slices.
+struct grpc_slice_refcount {
+ public:
+  enum class Type {
+    STATIC,    // Refcount for a static metadata slice.
+    INTERNED,  // Refcount for an interned slice.
+    NOP,       // No-Op
+    REGULAR    // Refcount for non-static-metadata, non-interned slices.
+  };
+  typedef void (*DestroyerFn)(void*);
+
+  grpc_slice_refcount() = default;
+
+  explicit grpc_slice_refcount(Type t) : ref_type_(t) {}
+
+  explicit grpc_slice_refcount(grpc_slice_refcount* sub) : sub_refcount_(sub) {}
+  // Regular constructor for grpc_slice_refcount.
+  //
+  // Parameters:
+  //  1. grpc_slice_refcount::Type type
+  //  Whether we are the refcount for a static
+  //  metadata slice, an interned slice, or any other kind of slice.
+  //
+  //  2. RefCount* ref
+  //  The pointer to the actual underlying grpc_core::RefCount. Rather than
+  //  performing struct offset computations as in the original implementation to
+  //  get to the refcount, which requires a virtual method, we devirtualize by
+  //  using a nullable pointer to allow a single pair of Ref/Unref methods.
+  //
+  //  3. DestroyerFn destroyer_fn
+  //  Called when the refcount goes to 0, with destroyer_arg as parameter.
+  //
+  //  4. void* destroyer_arg
+  //  Argument for the virtualized destructor.
+  //
+  //  5. grpc_slice_refcount* sub
+  //  Argument used for interned slices.
+  grpc_slice_refcount(grpc_slice_refcount::Type type, grpc_core::RefCount* ref,
+                      DestroyerFn destroyer_fn, void* destroyer_arg,
+                      grpc_slice_refcount* sub)
+      : ref_(ref),
+        ref_type_(type),
+        sub_refcount_(sub),
+        dest_fn_(destroyer_fn),
+        destroy_fn_arg_(destroyer_arg) {}
+  // Initializer for static refcounts.
+  grpc_slice_refcount(grpc_slice_refcount* sub, Type type)
+      : ref_type_(type), sub_refcount_(sub) {}
+
+  Type GetType() const { return ref_type_; }
+
+  int Eq(const grpc_slice& a, const grpc_slice& b);
+
+  uint32_t Hash(const grpc_slice& slice);
+  void Ref() {
+    if (ref_ == nullptr) return;
+    ref_->RefNonZero();
+  }
+  void Unref() {
+    if (ref_ == nullptr) return;
+    if (ref_->Unref()) {
+      dest_fn_(destroy_fn_arg_);
+    }
+  }
+
+  grpc_slice_refcount* sub_refcount() const { return sub_refcount_; }
+
+ private:
+  grpc_core::RefCount* ref_ = nullptr;
+  const Type ref_type_ = Type::REGULAR;
+  grpc_slice_refcount* sub_refcount_ = this;
+  DestroyerFn dest_fn_ = nullptr;
+  void* destroy_fn_arg_ = nullptr;
+};
+
+namespace grpc_core {
+
+struct StaticSliceRefcount {
+  static grpc_slice_refcount kStaticSubRefcount;
+
+  StaticSliceRefcount(uint32_t index)
+      : base(&kStaticSubRefcount, grpc_slice_refcount::Type::STATIC),
+        index(index) {}
+
+  grpc_slice_refcount base;
+  const uint32_t index;
+};
+
+extern grpc_slice_refcount kNoopRefcount;
+
+struct InternedSliceRefcount {
+  static void Destroy(void* arg) {
+    auto* rc = static_cast<InternedSliceRefcount*>(arg);
+    rc->~InternedSliceRefcount();
+    gpr_free(rc);
+  }
+
+  InternedSliceRefcount(size_t length, uint32_t hash,
+                        InternedSliceRefcount* bucket_next)
+      : base(grpc_slice_refcount::Type::INTERNED, &refcnt, Destroy, this, &sub),
+        sub(grpc_slice_refcount::Type::REGULAR, &refcnt, Destroy, this, &sub),
+        length(length),
+        hash(hash),
+        bucket_next(bucket_next) {}
+
+  ~InternedSliceRefcount();
+
+  grpc_slice_refcount base;
+  grpc_slice_refcount sub;
+  const size_t length;
+  RefCount refcnt;
+  const uint32_t hash;
+  InternedSliceRefcount* bucket_next;
+};
+
+}  // namespace grpc_core
+
+inline size_t grpc_refcounted_slice_length(const grpc_slice& slice) {
+  GPR_DEBUG_ASSERT(slice.refcount != nullptr);
+  return slice.data.refcounted.length;
+}
+
+inline const uint8_t* grpc_refcounted_slice_data(const grpc_slice& slice) {
+  GPR_DEBUG_ASSERT(slice.refcount != nullptr);
+  return slice.data.refcounted.bytes;
+}
+
+inline int grpc_slice_refcount::Eq(const grpc_slice& a, const grpc_slice& b) {
+  GPR_DEBUG_ASSERT(a.refcount != nullptr);
+  GPR_DEBUG_ASSERT(a.refcount == this);
+  switch (ref_type_) {
+    case Type::STATIC:
+      GPR_DEBUG_ASSERT(
+          (GRPC_STATIC_METADATA_INDEX(a) == GRPC_STATIC_METADATA_INDEX(b)) ==
+          (a.refcount == b.refcount));
+    case Type::INTERNED:
+      return a.refcount == b.refcount;
+    case Type::NOP:
+    case Type::REGULAR:
+      break;
+  }
+  if (grpc_refcounted_slice_length(a) != GRPC_SLICE_LENGTH(b)) return false;
+  if (grpc_refcounted_slice_length(a) == 0) return true;
+  return 0 == memcmp(grpc_refcounted_slice_data(a), GRPC_SLICE_START_PTR(b),
+                     grpc_refcounted_slice_length(a));
+}
+
+inline uint32_t grpc_slice_refcount::Hash(const grpc_slice& slice) {
+  GPR_DEBUG_ASSERT(slice.refcount != nullptr);
+  GPR_DEBUG_ASSERT(slice.refcount == this);
+  switch (ref_type_) {
+    case Type::STATIC:
+      return ::grpc_static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(
+          slice)];
+    case Type::INTERNED:
+      return reinterpret_cast<grpc_core::InternedSliceRefcount*>(slice.refcount)
+          ->hash;
+    case Type::NOP:
+    case Type::REGULAR:
+      break;
+  }
+  return gpr_murmur_hash3(grpc_refcounted_slice_data(slice),
+                          grpc_refcounted_slice_length(slice),
+                          grpc_core::g_hash_seed);
+}
+
+inline const grpc_slice& grpc_slice_ref_internal(const grpc_slice& slice) {
+  if (slice.refcount) {
+    slice.refcount->Ref();
+  }
+  return slice;
+}
+
+inline void grpc_slice_unref_internal(const grpc_slice& slice) {
+  if (slice.refcount) {
+    slice.refcount->Unref();
+  }
+}
+
+void grpc_slice_buffer_reset_and_unref_internal(grpc_slice_buffer* sb);
+void grpc_slice_buffer_partial_unref_internal(grpc_slice_buffer* sb,
+                                              size_t idx);
+void grpc_slice_buffer_destroy_internal(grpc_slice_buffer* sb);
+
+// Returns a pointer to the first slice in the slice buffer without giving
+// ownership to or a reference count on that slice.
+inline grpc_slice* grpc_slice_buffer_peek_first(grpc_slice_buffer* sb) {
+  GPR_DEBUG_ASSERT(sb->count > 0);
+  return &sb->slices[0];
+}
+
+// Removes the first slice from the slice buffer.
+void grpc_slice_buffer_remove_first(grpc_slice_buffer* sb);
+
+// Calls grpc_slice_sub with the given parameters on the first slice.
+void grpc_slice_buffer_sub_first(grpc_slice_buffer* sb, size_t begin,
+                                 size_t end);
+
+/* Check if a slice is interned */
+bool grpc_slice_is_interned(const grpc_slice& slice);
+inline bool grpc_slice_is_interned(const grpc_slice& slice) {
+  return (slice.refcount &&
+          (slice.refcount->GetType() == grpc_slice_refcount::Type::INTERNED ||
+           slice.refcount->GetType() == grpc_slice_refcount::Type::STATIC));
+}
+
+inline bool grpc_slice_static_interned_equal(const grpc_slice& a,
+                                             const grpc_slice& b) {
+  GPR_DEBUG_ASSERT(grpc_slice_is_interned(a) && grpc_slice_is_interned(b));
+  return a.refcount == b.refcount;
+}
+
+void grpc_slice_intern_init(void);
+void grpc_slice_intern_shutdown(void);
+void grpc_test_only_set_slice_hash_seed(uint32_t key);
+// if slice matches a static slice, returns the static slice
+// otherwise returns the passed in slice (without reffing it)
+// used for surface boundaries where we might receive an un-interned static
+// string
+grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
+                                          bool* returned_slice_is_different);
+uint32_t grpc_static_slice_hash(grpc_slice s);
+int grpc_static_slice_eq(grpc_slice a, grpc_slice b);
+
+inline uint32_t grpc_slice_hash_refcounted(const grpc_slice& s) {
+  GPR_DEBUG_ASSERT(s.refcount != nullptr);
+  return s.refcount->Hash(s);
+}
+
+inline uint32_t grpc_slice_default_hash_internal(const grpc_slice& s) {
+  return gpr_murmur_hash3(GRPC_SLICE_START_PTR(s), GRPC_SLICE_LENGTH(s),
+                          grpc_core::g_hash_seed);
+}
+
+inline uint32_t grpc_slice_hash_internal(const grpc_slice& s) {
+  return s.refcount == nullptr ? grpc_slice_default_hash_internal(s)
+                               : grpc_slice_hash_refcounted(s);
+}
+
+grpc_slice grpc_slice_from_moved_buffer(grpc_core::UniquePtr<char> p,
+                                        size_t len);
+grpc_slice grpc_slice_from_moved_string(grpc_core::UniquePtr<char> p);
+
+// Returns the memory used by this slice, not counting the slice structure
+// itself. This means that inlined and slices from static strings will return
+// 0. All other slices will return the size of the allocated chars.
+size_t grpc_slice_memory_usage(grpc_slice s);
+
+grpc_core::UnmanagedMemorySlice grpc_slice_sub_no_ref(
+    const grpc_core::UnmanagedMemorySlice& source, size_t begin, size_t end);
+
+namespace grpc_core {
+
+struct SliceHash {
+  std::size_t operator()(const grpc_slice& slice) const {
+    return grpc_slice_hash_internal(slice);
+  }
+};
+
+}  // namespace grpc_core
+
+inline bool operator==(const grpc_slice& s1, const grpc_slice& s2) {
+  return grpc_slice_eq(s1, s2);
+}
+
+#endif /* GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H */