grpc 1.21.0 → 1.22.0

data/src/core/lib/surface/completion_queue.h

@@ -77,7 +77,8 @@ bool grpc_cq_begin_op(grpc_completion_queue* cc, void* tag);
    grpc_cq_begin_op */
 void grpc_cq_end_op(grpc_completion_queue* cc, void* tag, grpc_error* error,
                     void (*done)(void* done_arg, grpc_cq_completion* storage),
-                    void* done_arg, grpc_cq_completion* storage);
+                    void* done_arg, grpc_cq_completion* storage,
+                    bool internal = false);
 
 grpc_pollset* grpc_cq_pollset(grpc_completion_queue* cc);
 

data/src/core/lib/surface/server.cc

@@ -513,7 +513,7 @@ static void publish_call(grpc_server* server, call_data* calld, size_t cq_idx,
   }
 
   grpc_cq_end_op(calld->cq_new, rc->tag, GRPC_ERROR_NONE, done_request_event,
-                 rc, &rc->completion);
+                 rc, &rc->completion, true);
 }
 
 static void publish_new_rpc(void* arg, grpc_error* error) {
@@ -625,8 +625,8 @@ static void start_new_rpc(grpc_call_element* elem) {
   if (chand->registered_methods && calld->path_set && calld->host_set) {
     /* TODO(ctiller): unify these two searches */
     /* check for an exact match with host */
-    hash = GRPC_MDSTR_KV_HASH(grpc_slice_hash(calld->host),
-                              grpc_slice_hash(calld->path));
+    hash = GRPC_MDSTR_KV_HASH(grpc_slice_hash_internal(calld->host),
+                              grpc_slice_hash_internal(calld->path));
     for (i = 0; i <= chand->registered_method_max_probes; i++) {
       rm = &chand->registered_methods[(hash + i) %
                                       chand->registered_method_slots];
@@ -644,7 +644,7 @@ static void start_new_rpc(grpc_call_element* elem) {
       return;
     }
     /* check for a wildcard method definition (no host set) */
-    hash = GRPC_MDSTR_KV_HASH(0, grpc_slice_hash(calld->path));
+    hash = GRPC_MDSTR_KV_HASH(0, grpc_slice_hash_internal(calld->path));
     for (i = 0; i <= chand->registered_method_max_probes; i++) {
       rm = &chand->registered_methods[(hash + i) %
                                       chand->registered_method_slots];
@@ -1194,14 +1194,14 @@ void grpc_server_setup_transport(
       bool has_host;
       grpc_slice method;
       if (rm->host != nullptr) {
-        host = grpc_slice_intern(grpc_slice_from_static_string(rm->host));
+        host = grpc_slice_from_static_string(rm->host);
         has_host = true;
       } else {
         has_host = false;
       }
-      method = grpc_slice_intern(grpc_slice_from_static_string(rm->method));
-      hash = GRPC_MDSTR_KV_HASH(has_host ? grpc_slice_hash(host) : 0,
-                                grpc_slice_hash(method));
+      method = grpc_slice_from_static_string(rm->method);
+      hash = GRPC_MDSTR_KV_HASH(has_host ? grpc_slice_hash_internal(host) : 0,
+                                grpc_slice_hash_internal(method));
       for (probes = 0; chand->registered_methods[(hash + probes) % slots]
                            .server_registered_method != nullptr;
            probes++)

data/src/core/lib/surface/validate_metadata.cc

@@ -24,12 +24,14 @@
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 
+#include "src/core/lib/gprpp/memory.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
 #include "src/core/lib/surface/validate_metadata.h"
 
-static grpc_error* conforms_to(grpc_slice slice, const uint8_t* legal_bits,
+static grpc_error* conforms_to(const grpc_slice& slice,
+                               const uint8_t* legal_bits,
                                const char* err_desc) {
   const uint8_t* p = GRPC_SLICE_START_PTR(slice);
   const uint8_t* e = GRPC_SLICE_END_PTR(slice);
@@ -38,13 +40,12 @@ static grpc_error* conforms_to(grpc_slice slice, const uint8_t* legal_bits,
     int byte = idx / 8;
     int bit = idx % 8;
     if ((legal_bits[byte] & (1 << bit)) == 0) {
-      char* dump = grpc_dump_slice(slice, GPR_DUMP_HEX | GPR_DUMP_ASCII);
       grpc_error* error = grpc_error_set_str(
           grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(err_desc),
                              GRPC_ERROR_INT_OFFSET,
                              p - GRPC_SLICE_START_PTR(slice)),
-          GRPC_ERROR_STR_RAW_BYTES, grpc_slice_from_copied_string(dump));
-      gpr_free(dump);
+          GRPC_ERROR_STR_RAW_BYTES,
+          grpc_dump_slice_to_slice(slice, GPR_DUMP_HEX | GPR_DUMP_ASCII));
       return error;
     }
   }
@@ -57,7 +58,7 @@ static int error2int(grpc_error* error) {
   return r;
 }
 
-grpc_error* grpc_validate_header_key_is_legal(grpc_slice slice) {
+grpc_error* grpc_validate_header_key_is_legal(const grpc_slice& slice) {
   static const uint8_t legal_header_bits[256 / 8] = {
       0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0xff, 0x03, 0x00, 0x00, 0x00,
       0x80, 0xfe, 0xff, 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -77,7 +78,8 @@ int grpc_header_key_is_legal(grpc_slice slice) {
   return error2int(grpc_validate_header_key_is_legal(slice));
 }
 
-grpc_error* grpc_validate_header_nonbin_value_is_legal(grpc_slice slice) {
+grpc_error* grpc_validate_header_nonbin_value_is_legal(
+    const grpc_slice& slice) {
   static const uint8_t legal_header_bits[256 / 8] = {
       0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
       0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -89,7 +91,11 @@ int grpc_header_nonbin_value_is_legal(grpc_slice slice) {
   return error2int(grpc_validate_header_nonbin_value_is_legal(slice));
 }
 
+int grpc_is_binary_header_internal(const grpc_slice& slice) {
+  return grpc_key_is_binary_header(GRPC_SLICE_START_PTR(slice),
+                                   GRPC_SLICE_LENGTH(slice));
+}
+
 int grpc_is_binary_header(grpc_slice slice) {
-  if (GRPC_SLICE_LENGTH(slice) < 5) return 0;
-  return 0 == memcmp(GRPC_SLICE_END_PTR(slice) - 4, "-bin", 4);
+  return grpc_is_binary_header_internal(slice);
 }

data/src/core/lib/surface/validate_metadata.h

@@ -24,7 +24,18 @@
 #include <grpc/slice.h>
 #include "src/core/lib/iomgr/error.h"
 
-grpc_error* grpc_validate_header_key_is_legal(grpc_slice slice);
-grpc_error* grpc_validate_header_nonbin_value_is_legal(grpc_slice slice);
+grpc_error* grpc_validate_header_key_is_legal(const grpc_slice& slice);
+grpc_error* grpc_validate_header_nonbin_value_is_legal(const grpc_slice& slice);
+
+int grpc_is_binary_header_internal(const grpc_slice& slice);
+inline int grpc_key_is_binary_header(const uint8_t* buf, size_t length) {
+  if (length < 5) return 0;
+  return 0 == memcmp(buf + length - 4, "-bin", 4);
+}
+inline int grpc_is_refcounted_slice_binary_header(const grpc_slice& slice) {
+  GPR_DEBUG_ASSERT(slice.refcount != nullptr);
+  return grpc_key_is_binary_header(slice.data.refcounted.bytes,
+                                   slice.data.refcounted.length);
+}
 
 #endif /* GRPC_CORE_LIB_SURFACE_VALIDATE_METADATA_H */

data/src/core/lib/surface/version.cc

@@ -25,4 +25,4 @@
 
 const char* grpc_version_string(void) { return "7.0.0"; }
 
-const char* grpc_g_stands_for(void) { return "gandalf"; }
+const char* grpc_g_stands_for(void) { return "gale"; }

data/src/core/lib/transport/metadata.cc

@@ -43,6 +43,7 @@
 
 using grpc_core::AllocatedMetadata;
 using grpc_core::InternedMetadata;
+using grpc_core::StaticMetadata;
 using grpc_core::UserData;
 
 /* There are two kinds of mdelem and mdstr instances.
@@ -100,15 +101,20 @@ void grpc_mdelem_trace_unref(void* md, const grpc_slice& key,
 #define TABLE_IDX(hash, capacity) (((hash) >> (LOG2_SHARD_COUNT)) % (capacity))
 #define SHARD_IDX(hash) ((hash) & ((1 << (LOG2_SHARD_COUNT)) - 1))
 
+void StaticMetadata::HashInit() {
+  uint32_t k_hash = grpc_slice_hash_internal(kv_.key);
+  uint32_t v_hash = grpc_slice_hash_internal(kv_.value);
+  hash_ = GRPC_MDSTR_KV_HASH(k_hash, v_hash);
+}
+
 AllocatedMetadata::AllocatedMetadata(const grpc_slice& key,
                                      const grpc_slice& value)
-    : key_(grpc_slice_ref_internal(key)),
-      value_(grpc_slice_ref_internal(value)),
-      refcnt_(1) {
+    : RefcountedMdBase(grpc_slice_ref_internal(key),
+                       grpc_slice_ref_internal(value)) {
 #ifndef NDEBUG
   if (grpc_trace_metadata.enabled()) {
-    char* key_str = grpc_slice_to_c_string(key_);
-    char* value_str = grpc_slice_to_c_string(value_);
+    char* key_str = grpc_slice_to_c_string(key);
+    char* value_str = grpc_slice_to_c_string(value);
     gpr_log(GPR_DEBUG, "ELM ALLOC:%p:%" PRIdPTR ": '%s' = '%s'", this,
             RefValue(), key_str, value_str);
     gpr_free(key_str);
@@ -118,8 +124,8 @@ AllocatedMetadata::AllocatedMetadata(const grpc_slice& key,
 }
 
 AllocatedMetadata::~AllocatedMetadata() {
-  grpc_slice_unref_internal(key_);
-  grpc_slice_unref_internal(value_);
+  grpc_slice_unref_internal(key());
+  grpc_slice_unref_internal(value());
   void* user_data = user_data_.data.Load(grpc_core::MemoryOrder::RELAXED);
   if (user_data) {
     destroy_user_data_func destroy_user_data =
@@ -131,15 +137,13 @@ AllocatedMetadata::~AllocatedMetadata() {
 InternedMetadata::InternedMetadata(const grpc_slice& key,
                                    const grpc_slice& value, uint32_t hash,
                                    InternedMetadata* next)
-    : key_(grpc_slice_ref_internal(key)),
-      value_(grpc_slice_ref_internal(value)),
-      refcnt_(1),
-      hash_(hash),
+    : RefcountedMdBase(grpc_slice_ref_internal(key),
+                       grpc_slice_ref_internal(value), hash),
       link_(next) {
 #ifndef NDEBUG
   if (grpc_trace_metadata.enabled()) {
-    char* key_str = grpc_slice_to_c_string(key_);
-    char* value_str = grpc_slice_to_c_string(value_);
+    char* key_str = grpc_slice_to_c_string(key);
+    char* value_str = grpc_slice_to_c_string(value);
     gpr_log(GPR_DEBUG, "ELM   NEW:%p:%" PRIdPTR ": '%s' = '%s'", this,
             RefValue(), key_str, value_str);
     gpr_free(key_str);
@@ -149,8 +153,8 @@ InternedMetadata::InternedMetadata(const grpc_slice& key,
 }
 
 InternedMetadata::~InternedMetadata() {
-  grpc_slice_unref_internal(key_);
-  grpc_slice_unref_internal(value_);
+  grpc_slice_unref_internal(key());
+  grpc_slice_unref_internal(value());
   void* user_data = user_data_.data.Load(grpc_core::MemoryOrder::RELAXED);
   if (user_data) {
     destroy_user_data_func destroy_user_data =
@@ -223,17 +227,20 @@ void grpc_mdctx_global_shutdown() {
   }
 }
 
+#ifndef NDEBUG
 static int is_mdelem_static(grpc_mdelem e) {
-  return GRPC_MDELEM_DATA(e) >= &grpc_static_mdelem_table[0] &&
-         GRPC_MDELEM_DATA(e) <
+  return reinterpret_cast<grpc_core::StaticMetadata*>(GRPC_MDELEM_DATA(e)) >=
+             &grpc_static_mdelem_table[0] &&
+         reinterpret_cast<grpc_core::StaticMetadata*>(GRPC_MDELEM_DATA(e)) <
              &grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
 }
+#endif
 
 void InternedMetadata::RefWithShardLocked(mdtab_shard* shard) {
 #ifndef NDEBUG
   if (grpc_trace_metadata.enabled()) {
-    char* key_str = grpc_slice_to_c_string(key_);
-    char* value_str = grpc_slice_to_c_string(value_);
+    char* key_str = grpc_slice_to_c_string(key());
+    char* value_str = grpc_slice_to_c_string(value());
     intptr_t value = RefValue();
     gpr_log(__FILE__, __LINE__, GPR_LOG_SEVERITY_DEBUG,
             "ELM   REF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", this, value,
@@ -323,8 +330,8 @@ grpc_mdelem grpc_mdelem_create(
     }
   }
 
-  uint32_t hash =
-      GRPC_MDSTR_KV_HASH(grpc_slice_hash(key), grpc_slice_hash(value));
+  uint32_t hash = GRPC_MDSTR_KV_HASH(grpc_slice_hash_refcounted(key),
+                                     grpc_slice_hash_refcounted(value));
   InternedMetadata* md;
   mdtab_shard* shard = &g_shards[SHARD_IDX(hash)];
   size_t idx;
@@ -391,8 +398,11 @@ void* grpc_mdelem_get_user_data(grpc_mdelem md, void (*destroy_func)(void*)) {
     case GRPC_MDELEM_STORAGE_EXTERNAL:
       return nullptr;
     case GRPC_MDELEM_STORAGE_STATIC:
-      return (void*)grpc_static_mdelem_user_data[GRPC_MDELEM_DATA(md) -
-                                                 grpc_static_mdelem_table];
+      return reinterpret_cast<void*>(
+          grpc_static_mdelem_user_data
+              [reinterpret_cast<grpc_core::StaticMetadata*>(
+                   GRPC_MDELEM_DATA(md)) -
+               grpc_static_mdelem_table]);
     case GRPC_MDELEM_STORAGE_ALLOCATED: {
       auto* am = reinterpret_cast<AllocatedMetadata*>(GRPC_MDELEM_DATA(md));
       return get_user_data(am->user_data(), destroy_func);
@@ -430,15 +440,18 @@ void* grpc_mdelem_set_user_data(grpc_mdelem md, void (*destroy_func)(void*),
       return nullptr;
     case GRPC_MDELEM_STORAGE_STATIC:
       destroy_func(data);
-      return (void*)grpc_static_mdelem_user_data[GRPC_MDELEM_DATA(md) -
-                                                 grpc_static_mdelem_table];
+      return reinterpret_cast<void*>(
+          grpc_static_mdelem_user_data
+              [reinterpret_cast<grpc_core::StaticMetadata*>(
+                   GRPC_MDELEM_DATA(md)) -
+               grpc_static_mdelem_table]);
     case GRPC_MDELEM_STORAGE_ALLOCATED: {
       auto* am = reinterpret_cast<AllocatedMetadata*>(GRPC_MDELEM_DATA(md));
       return set_user_data(am->user_data(), destroy_func, data);
     }
     case GRPC_MDELEM_STORAGE_INTERNED: {
       auto* im = reinterpret_cast<InternedMetadata*> GRPC_MDELEM_DATA(md);
-      GPR_ASSERT(!is_mdelem_static(md));
+      GPR_DEBUG_ASSERT(!is_mdelem_static(md));
       return set_user_data(im->user_data(), destroy_func, data);
     }
   }
@@ -482,3 +495,20 @@ void grpc_mdelem_do_unref(grpc_mdelem gmd DEBUG_ARGS) {
     }
   }
 }
+
+void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
+                                uint32_t hash DEBUG_ARGS) {
+  switch (storage) {
+    case GRPC_MDELEM_STORAGE_EXTERNAL:
+    case GRPC_MDELEM_STORAGE_STATIC:
+      return;
+    case GRPC_MDELEM_STORAGE_INTERNED: {
+      note_disposed_interned_metadata(hash);
+      break;
+    }
+    case GRPC_MDELEM_STORAGE_ALLOCATED: {
+      grpc_core::Delete(reinterpret_cast<AllocatedMetadata*>(ptr));
+      break;
+    }
+  }
+}

data/src/core/lib/transport/metadata.h

@@ -21,7 +21,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "include/grpc/impl/codegen/log.h"
+#include <grpc/impl/codegen/log.h>
 
 #include <grpc/grpc.h>
 #include <grpc/slice.h>
@@ -30,6 +30,7 @@
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/atomic.h"
 #include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/slice/slice_utils.h"
 
 extern grpc_core::DebugOnlyTraceFlag grpc_trace_metadata;
 
@@ -79,17 +80,22 @@ typedef struct grpc_mdelem_data {
    this bit set in their integer value */
 #define GRPC_MDELEM_STORAGE_INTERNED_BIT 1
 
+/* External and static storage metadata has no refcount to ref/unref. Allocated
+ * and interned metadata do have a refcount. Metadata ref and unref methods use
+ * a switch statement on this enum to determine which behaviour to execute.
+ * Keeping the no-ref cases together and the ref-cases together leads to
+ * slightly better code generation (9 inlined instructions rather than 10). */
 typedef enum {
   /* memory pointed to by grpc_mdelem::payload is owned by an external system */
   GRPC_MDELEM_STORAGE_EXTERNAL = 0,
-  /* memory pointed to by grpc_mdelem::payload is interned by the metadata
-     system */
-  GRPC_MDELEM_STORAGE_INTERNED = GRPC_MDELEM_STORAGE_INTERNED_BIT,
+  /* memory is in the static metadata table */
+  GRPC_MDELEM_STORAGE_STATIC = GRPC_MDELEM_STORAGE_INTERNED_BIT,
   /* memory pointed to by grpc_mdelem::payload is allocated by the metadata
      system */
   GRPC_MDELEM_STORAGE_ALLOCATED = 2,
-  /* memory is in the static metadata table */
-  GRPC_MDELEM_STORAGE_STATIC = 2 | GRPC_MDELEM_STORAGE_INTERNED_BIT,
+  /* memory pointed to by grpc_mdelem::payload is interned by the metadata
+     system */
+  GRPC_MDELEM_STORAGE_INTERNED = 2 | GRPC_MDELEM_STORAGE_INTERNED_BIT,
 } grpc_mdelem_data_storage;
 
 struct grpc_mdelem {
@@ -138,7 +144,17 @@ bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b);
  * grpc_mdelem_eq and remove unnecessary checks. */
 inline bool grpc_mdelem_static_value_eq(grpc_mdelem a, grpc_mdelem b_static) {
   if (a.payload == b_static.payload) return true;
-  return grpc_slice_eq(GRPC_MDVALUE(a), GRPC_MDVALUE(b_static));
+  return grpc_slice_eq_static_interned(GRPC_MDVALUE(a), GRPC_MDVALUE(b_static));
+}
+#define GRPC_MDISNULL(md) (GRPC_MDELEM_DATA(md) == NULL)
+
+inline bool grpc_mdelem_both_interned_eq(grpc_mdelem a_interned,
+                                         grpc_mdelem b_interned) {
+  GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(a_interned) ||
+                   GRPC_MDISNULL(a_interned));
+  GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(b_interned) ||
+                   GRPC_MDISNULL(b_interned));
+  return a_interned.payload == b_interned.payload;
 }
 
 /* Mutator and accessor for grpc_mdelem user data. The destructor function
@@ -168,17 +184,34 @@ struct UserData {
   grpc_core::Atomic<void*> data;
 };
 
-class InternedMetadata {
+class StaticMetadata {
  public:
-  struct BucketLink {
-    explicit BucketLink(InternedMetadata* md) : next(md) {}
+  StaticMetadata(const grpc_slice& key, const grpc_slice& value)
+      : kv_({key, value}), hash_(0) {}
 
-    InternedMetadata* next = nullptr;
-  };
+  const grpc_mdelem_data& data() const { return kv_; }
 
-  InternedMetadata(const grpc_slice& key, const grpc_slice& value,
-                   uint32_t hash, InternedMetadata* next);
-  ~InternedMetadata();
+  void HashInit();
+  uint32_t hash() { return hash_; }
+
+ private:
+  grpc_mdelem_data kv_;
+
+  /* private only data */
+  uint32_t hash_;
+};
+
+class RefcountedMdBase {
+ public:
+  RefcountedMdBase(const grpc_slice& key, const grpc_slice& value)
+      : key_(key), value_(value), refcnt_(1) {}
+  RefcountedMdBase(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash)
+      : key_(key), value_(value), refcnt_(1), hash_(hash) {}
+
+  const grpc_slice& key() const { return key_; }
+  const grpc_slice& value() const { return value_; }
+  uint32_t hash() { return hash_; }
 
 #ifndef NDEBUG
   void Ref(const char* file, int line) {
@@ -190,92 +223,65 @@ class InternedMetadata {
     grpc_mdelem_trace_unref(this, key_, value_, RefValue(), file, line);
     return Unref();
   }
-#else
-  // We define a naked Ref() in the else-clause to make sure we don't
-  // inadvertently skip the assert on debug builds.
+#endif
   void Ref() {
     /* we can assume the ref count is >= 1 as the application is calling
        this function - meaning that no adjustment to mdtab_free is necessary,
        simplifying the logic here to be just an atomic increment */
     refcnt_.FetchAdd(1, MemoryOrder::RELAXED);
   }
-#endif  // ifndef NDEBUG
   bool Unref() {
     const intptr_t prior = refcnt_.FetchSub(1, MemoryOrder::ACQ_REL);
     GPR_DEBUG_ASSERT(prior > 0);
     return prior == 1;
   }
 
-  void RefWithShardLocked(mdtab_shard* shard);
-  const grpc_slice& key() const { return key_; }
-  const grpc_slice& value() const { return value_; }
-  UserData* user_data() { return &user_data_; }
-  uint32_t hash() { return hash_; }
-  InternedMetadata* bucket_next() { return link_.next; }
-  void set_bucket_next(InternedMetadata* md) { link_.next = md; }
-
-  static size_t CleanupLinkedMetadata(BucketLink* head);
-
- private:
+ protected:
+  intptr_t RefValue() { return refcnt_.Load(MemoryOrder::RELAXED); }
   bool AllRefsDropped() { return refcnt_.Load(MemoryOrder::ACQUIRE) == 0; }
   bool FirstRef() { return refcnt_.FetchAdd(1, MemoryOrder::RELAXED) == 0; }
-  intptr_t RefValue() { return refcnt_.Load(MemoryOrder::RELAXED); }
 
+ private:
   /* must be byte compatible with grpc_mdelem_data */
   grpc_slice key_;
   grpc_slice value_;
-
-  /* private only data */
   grpc_core::Atomic<intptr_t> refcnt_;
-  uint32_t hash_;
+  uint32_t hash_ = 0;
+};
 
-  UserData user_data_;
+class InternedMetadata : public RefcountedMdBase {
+ public:
+  struct BucketLink {
+    explicit BucketLink(InternedMetadata* md) : next(md) {}
+
+    InternedMetadata* next = nullptr;
+  };
+
+  InternedMetadata(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash, InternedMetadata* next);
+  ~InternedMetadata();
+
+  void RefWithShardLocked(mdtab_shard* shard);
+  UserData* user_data() { return &user_data_; }
+  InternedMetadata* bucket_next() { return link_.next; }
+  void set_bucket_next(InternedMetadata* md) { link_.next = md; }
 
+  static size_t CleanupLinkedMetadata(BucketLink* head);
+
+ private:
+  UserData user_data_;
   BucketLink link_;
 };
 
 /* Shadow structure for grpc_mdelem_data for allocated elements */
-class AllocatedMetadata {
+class AllocatedMetadata : public RefcountedMdBase {
  public:
   AllocatedMetadata(const grpc_slice& key, const grpc_slice& value);
   ~AllocatedMetadata();
 
-  const grpc_slice& key() const { return key_; }
-  const grpc_slice& value() const { return value_; }
   UserData* user_data() { return &user_data_; }
 
-#ifndef NDEBUG
-  void Ref(const char* file, int line) {
-    grpc_mdelem_trace_ref(this, key_, value_, RefValue(), file, line);
-    Ref();
-  }
-  bool Unref(const char* file, int line) {
-    grpc_mdelem_trace_unref(this, key_, value_, RefValue(), file, line);
-    return Unref();
-  }
-#endif  // ifndef NDEBUG
-  void Ref() {
-    /* we can assume the ref count is >= 1 as the application is calling
-       this function - meaning that no adjustment to mdtab_free is necessary,
-       simplifying the logic here to be just an atomic increment */
-    refcnt_.FetchAdd(1, MemoryOrder::RELAXED);
-  }
-  bool Unref() {
-    const intptr_t prior = refcnt_.FetchSub(1, MemoryOrder::ACQ_REL);
-    GPR_DEBUG_ASSERT(prior > 0);
-    return prior == 1;
-  }
-
  private:
-  intptr_t RefValue() { return refcnt_.Load(MemoryOrder::RELAXED); }
-
-  /* must be byte compatible with grpc_mdelem_data */
-  grpc_slice key_;
-  grpc_slice value_;
-
-  /* private only data */
-  grpc_core::Atomic<intptr_t> refcnt_;
-
   UserData user_data_;
 };
 
@@ -320,30 +326,40 @@ inline grpc_mdelem grpc_mdelem_ref(grpc_mdelem gmd) {
 
 #ifndef NDEBUG
 #define GRPC_MDELEM_UNREF(s) grpc_mdelem_unref((s), __FILE__, __LINE__)
-void grpc_mdelem_do_unref(grpc_mdelem gmd, const char* file, int line);
+void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
+                                uint32_t hash, const char* file, int line);
 inline void grpc_mdelem_unref(grpc_mdelem gmd, const char* file, int line) {
 #else
 #define GRPC_MDELEM_UNREF(s) grpc_mdelem_unref((s))
-void grpc_mdelem_do_unref(grpc_mdelem gmd);
+void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
+                                uint32_t hash);
 inline void grpc_mdelem_unref(grpc_mdelem gmd) {
 #endif
-  switch (GRPC_MDELEM_STORAGE(gmd)) {
+  const grpc_mdelem_data_storage storage = GRPC_MDELEM_STORAGE(gmd);
+  switch (storage) {
     case GRPC_MDELEM_STORAGE_EXTERNAL:
     case GRPC_MDELEM_STORAGE_STATIC:
       return;
     case GRPC_MDELEM_STORAGE_INTERNED:
     case GRPC_MDELEM_STORAGE_ALLOCATED:
+      auto* md =
+          reinterpret_cast<grpc_core::RefcountedMdBase*> GRPC_MDELEM_DATA(gmd);
+      /* once the refcount hits zero, some other thread can come along and
+         free an interned md at any time: it's unsafe from this point on to
+         access it so we read the hash now. */
+      uint32_t hash = md->hash();
+      if (GPR_UNLIKELY(md->Unref())) {
 #ifndef NDEBUG
-      grpc_mdelem_do_unref(gmd, file, line);
+        grpc_mdelem_on_final_unref(storage, md, hash, file, line);
 #else
-      grpc_mdelem_do_unref(gmd);
+        grpc_mdelem_on_final_unref(storage, md, hash);
 #endif
+      }
       return;
   }
 }
 
 #define GRPC_MDNULL GRPC_MAKE_MDELEM(NULL, GRPC_MDELEM_STORAGE_EXTERNAL)
-#define GRPC_MDISNULL(md) (GRPC_MDELEM_DATA(md) == NULL)
 
 /* We add 32 bytes of padding as per RFC-7540 section 6.5.2. */
 #define GRPC_MDELEM_LENGTH(e)                                                  \