grpc 1.20.0 → 1.21.0

data/src/core/lib/surface/init.cc

@@ -33,7 +33,7 @@
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/fork.h"
-#include "src/core/lib/gprpp/mutex_lock.h"
+#include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/combiner.h"
@@ -73,6 +73,7 @@ static void do_basic_init(void) {
   g_shutting_down = false;
   grpc_register_built_in_plugins();
   grpc_cq_global_init();
+  gpr_time_init();
   g_initializations = 0;
 }
 
@@ -132,8 +133,6 @@ void grpc_init(void) {
     }
     grpc_core::Fork::GlobalInit();
     grpc_fork_handlers_auto_register();
-    gpr_time_init();
-    gpr_arena_init();
     grpc_stats_init();
     grpc_slice_intern_init();
     grpc_mdctx_global_init();
@@ -155,7 +154,7 @@ void grpc_init(void) {
      * at the appropriate time */
     grpc_register_security_filters();
     register_builtin_channel_init();
-    grpc_tracer_init("GRPC_TRACE");
+    grpc_tracer_init();
     /* no more changes to channel init pipelines */
     grpc_channel_init_finalize();
     grpc_iomgr_start();

data/src/core/lib/surface/lame_client.cc

@@ -39,7 +39,7 @@ namespace grpc_core {
 namespace {
 
 struct CallData {
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   grpc_linked_mdelem status;
   grpc_linked_mdelem details;
   grpc_core::Atomic<bool> filled_metadata;

data/src/core/lib/surface/server.cc

@@ -123,7 +123,7 @@ typedef struct shutdown_tag {
 typedef enum {
   /* waiting for metadata */
   NOT_STARTED,
-  /* inital metadata read, not flow controlled in yet */
+  /* initial metadata read, not flow controlled in yet */
   PENDING,
   /* flow controlled in, on completion queue */
   ACTIVATED,
@@ -190,7 +190,7 @@ struct call_data {
   grpc_closure publish;
 
   call_data* pending_next = nullptr;
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
 };
 
 struct request_matcher {
@@ -347,8 +347,8 @@ static void channel_broadcaster_shutdown(channel_broadcaster* cb,
  */
 
 static void request_matcher_init(request_matcher* rm, grpc_server* server) {
-  memset(rm, 0, sizeof(*rm));
   rm->server = server;
+  rm->pending_head = rm->pending_tail = nullptr;
   rm->requests_per_cq = static_cast<gpr_locked_mpscq*>(
       gpr_malloc(sizeof(*rm->requests_per_cq) * server->cq_count));
   for (size_t i = 0; i < server->cq_count; i++) {
@@ -464,7 +464,8 @@ static void destroy_channel(channel_data* chand, grpc_error* error) {
   GRPC_CLOSURE_INIT(&chand->finish_destroy_channel_closure,
                     finish_destroy_channel, chand, grpc_schedule_on_exec_ctx);
 
-  if (grpc_server_channel_trace.enabled() && error != GRPC_ERROR_NONE) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_server_channel_trace) &&
+      error != GRPC_ERROR_NONE) {
     const char* msg = grpc_error_string(error);
     gpr_log(GPR_INFO, "Disconnected client: %s", msg);
   }
@@ -601,8 +602,9 @@ static void finish_start_new_rpc(
       break;
     case GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER: {
       grpc_op op;
-      memset(&op, 0, sizeof(op));
       op.op = GRPC_OP_RECV_MESSAGE;
+      op.flags = 0;
+      op.reserved = nullptr;
       op.data.recv_message.recv_message = &calld->payload;
       GRPC_CLOSURE_INIT(&calld->publish, publish_new_rpc, elem,
                         grpc_schedule_on_exec_ctx);
@@ -1098,20 +1100,6 @@ void* grpc_server_register_method(
   return m;
 }
 
-static void start_listeners(void* s, grpc_error* error) {
-  grpc_server* server = static_cast<grpc_server*>(s);
-  for (listener* l = server->listeners; l; l = l->next) {
-    l->start(server, l->arg, server->pollsets, server->pollset_count);
-  }
-
-  gpr_mu_lock(&server->mu_global);
-  server->starting = false;
-  gpr_cv_signal(&server->starting_cv);
-  gpr_mu_unlock(&server->mu_global);
-
-  server_unref(server);
-}
-
 void grpc_server_start(grpc_server* server) {
   size_t i;
   grpc_core::ExecCtx exec_ctx;
@@ -1133,13 +1121,18 @@ void grpc_server_start(grpc_server* server) {
     request_matcher_init(&rm->matcher, server);
   }
 
-  server_ref(server);
+  gpr_mu_lock(&server->mu_global);
   server->starting = true;
-  GRPC_CLOSURE_SCHED(
-      GRPC_CLOSURE_CREATE(
-          start_listeners, server,
-          grpc_core::Executor::Scheduler(grpc_core::ExecutorJobType::SHORT)),
-      GRPC_ERROR_NONE);
+  gpr_mu_unlock(&server->mu_global);
+
+  for (listener* l = server->listeners; l; l = l->next) {
+    l->start(server, l->arg, server->pollsets, server->pollset_count);
+  }
+
+  gpr_mu_lock(&server->mu_global);
+  server->starting = false;
+  gpr_cv_signal(&server->starting_cv);
+  gpr_mu_unlock(&server->mu_global);
 }
 
 void grpc_server_get_pollsets(grpc_server* server, grpc_pollset*** pollsets,

data/src/core/lib/surface/version.cc

@@ -25,4 +25,4 @@
 
 const char* grpc_version_string(void) { return "7.0.0"; }
 
-const char* grpc_g_stands_for(void) { return "godric"; }
+const char* grpc_g_stands_for(void) { return "gandalf"; }

data/src/core/lib/transport/bdp_estimator.cc

@@ -46,7 +46,7 @@ grpc_millis BdpEstimator::CompletePing() {
               1e-9 * static_cast<double>(dt_ts.tv_nsec);
   double bw = dt > 0 ? (static_cast<double>(accumulator_) / dt) : 0;
   int start_inter_ping_delay = inter_ping_delay_;
-  if (grpc_bdp_estimator_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
     gpr_log(GPR_INFO,
             "bdp[%s]:complete acc=%" PRId64 " est=%" PRId64
             " dt=%lf bw=%lfMbs bw_est=%lfMbs",
@@ -57,7 +57,7 @@ grpc_millis BdpEstimator::CompletePing() {
   if (accumulator_ > 2 * estimate_ / 3 && bw > bw_est_) {
     estimate_ = GPR_MAX(accumulator_, estimate_ * 2);
     bw_est_ = bw;
-    if (grpc_bdp_estimator_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
       gpr_log(GPR_INFO, "bdp[%s]: estimate increased to %" PRId64, name_,
               estimate_);
     }
@@ -74,7 +74,7 @@ grpc_millis BdpEstimator::CompletePing() {
   }
   if (start_inter_ping_delay != inter_ping_delay_) {
     stable_estimate_count_ = 0;
-    if (grpc_bdp_estimator_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
       gpr_log(GPR_INFO, "bdp[%s]:update_inter_time to %dms", name_,
               inter_ping_delay_);
     }

data/src/core/lib/transport/bdp_estimator.h

@@ -49,7 +49,7 @@ class BdpEstimator {
   // grpc_bdp_estimator_add_incoming_bytes once a ping has been scheduled by a
   // transport (but not necessarily started)
   void SchedulePing() {
-    if (grpc_bdp_estimator_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
       gpr_log(GPR_INFO, "bdp[%s]:sched acc=%" PRId64 " est=%" PRId64, name_,
               accumulator_, estimate_);
     }
@@ -62,7 +62,7 @@ class BdpEstimator {
   // once
   // the ping is on the wire
   void StartPing() {
-    if (grpc_bdp_estimator_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
       gpr_log(GPR_INFO, "bdp[%s]:start acc=%" PRId64 " est=%" PRId64, name_,
               accumulator_, estimate_);
     }

data/src/core/lib/transport/connectivity_state.cc

@@ -48,7 +48,6 @@ void grpc_connectivity_state_init(grpc_connectivity_state_tracker* tracker,
                                   grpc_connectivity_state init_state,
                                   const char* name) {
   gpr_atm_no_barrier_store(&tracker->current_state_atm, init_state);
-  tracker->current_error = GRPC_ERROR_NONE;
   tracker->watchers = nullptr;
   tracker->name = gpr_strdup(name);
 }
@@ -69,7 +68,6 @@ void grpc_connectivity_state_destroy(grpc_connectivity_state_tracker* tracker) {
     GRPC_CLOSURE_SCHED(w->notify, error);
     gpr_free(w);
   }
-  GRPC_ERROR_UNREF(tracker->current_error);
   gpr_free(tracker->name);
 }
 
@@ -77,27 +75,13 @@ grpc_connectivity_state grpc_connectivity_state_check(
     grpc_connectivity_state_tracker* tracker) {
   grpc_connectivity_state cur = static_cast<grpc_connectivity_state>(
       gpr_atm_no_barrier_load(&tracker->current_state_atm));
-  if (grpc_connectivity_state_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
     gpr_log(GPR_INFO, "CONWATCH: %p %s: get %s", tracker, tracker->name,
             grpc_connectivity_state_name(cur));
   }
   return cur;
 }
 
-grpc_connectivity_state grpc_connectivity_state_get(
-    grpc_connectivity_state_tracker* tracker, grpc_error** error) {
-  grpc_connectivity_state cur = static_cast<grpc_connectivity_state>(
-      gpr_atm_no_barrier_load(&tracker->current_state_atm));
-  if (grpc_connectivity_state_trace.enabled()) {
-    gpr_log(GPR_INFO, "CONWATCH: %p %s: get %s", tracker, tracker->name,
-            grpc_connectivity_state_name(cur));
-  }
-  if (error != nullptr) {
-    *error = GRPC_ERROR_REF(tracker->current_error);
-  }
-  return cur;
-}
-
 bool grpc_connectivity_state_has_watchers(
     grpc_connectivity_state_tracker* connectivity_state) {
   return connectivity_state->watchers != nullptr;
@@ -108,7 +92,7 @@ bool grpc_connectivity_state_notify_on_state_change(
     grpc_closure* notify) {
   grpc_connectivity_state cur = static_cast<grpc_connectivity_state>(
       gpr_atm_no_barrier_load(&tracker->current_state_atm));
-  if (grpc_connectivity_state_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
     if (current == nullptr) {
       gpr_log(GPR_INFO, "CONWATCH: %p %s: unsubscribe notify=%p", tracker,
               tracker->name, notify);
@@ -140,7 +124,7 @@ bool grpc_connectivity_state_notify_on_state_change(
   } else {
     if (cur != *current) {
       *current = cur;
-      GRPC_CLOSURE_SCHED(notify, GRPC_ERROR_REF(tracker->current_error));
+      GRPC_CLOSURE_SCHED(notify, GRPC_ERROR_NONE);
     } else {
       grpc_connectivity_state_watcher* w =
           static_cast<grpc_connectivity_state_watcher*>(gpr_malloc(sizeof(*w)));
@@ -155,29 +139,15 @@ bool grpc_connectivity_state_notify_on_state_change(
 
 void grpc_connectivity_state_set(grpc_connectivity_state_tracker* tracker,
                                  grpc_connectivity_state state,
-                                 grpc_error* error, const char* reason) {
+                                 const char* reason) {
   grpc_connectivity_state cur = static_cast<grpc_connectivity_state>(
       gpr_atm_no_barrier_load(&tracker->current_state_atm));
   grpc_connectivity_state_watcher* w;
-  if (grpc_connectivity_state_trace.enabled()) {
-    const char* error_string = grpc_error_string(error);
-    gpr_log(GPR_INFO, "SET: %p %s: %s --> %s [%s] error=%p %s", tracker,
-            tracker->name, grpc_connectivity_state_name(cur),
-            grpc_connectivity_state_name(state), reason, error, error_string);
-  }
-  switch (state) {
-    case GRPC_CHANNEL_CONNECTING:
-    case GRPC_CHANNEL_IDLE:
-    case GRPC_CHANNEL_READY:
-      GPR_ASSERT(error == GRPC_ERROR_NONE);
-      break;
-    case GRPC_CHANNEL_SHUTDOWN:
-    case GRPC_CHANNEL_TRANSIENT_FAILURE:
-      GPR_ASSERT(error != GRPC_ERROR_NONE);
-      break;
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
+    gpr_log(GPR_INFO, "SET: %p %s: %s --> %s [%s]", tracker, tracker->name,
+            grpc_connectivity_state_name(cur),
+            grpc_connectivity_state_name(state), reason);
   }
-  GRPC_ERROR_UNREF(tracker->current_error);
-  tracker->current_error = error;
   if (cur == state) {
     return;
   }
@@ -186,10 +156,10 @@ void grpc_connectivity_state_set(grpc_connectivity_state_tracker* tracker,
   while ((w = tracker->watchers) != nullptr) {
     *w->current = state;
     tracker->watchers = w->next;
-    if (grpc_connectivity_state_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
       gpr_log(GPR_INFO, "NOTIFY: %p %s: %p", tracker, tracker->name, w->notify);
     }
-    GRPC_CLOSURE_SCHED(w->notify, GRPC_ERROR_REF(tracker->current_error));
+    GRPC_CLOSURE_SCHED(w->notify, GRPC_ERROR_NONE);
     gpr_free(w);
   }
 }

data/src/core/lib/transport/connectivity_state.h

@@ -37,8 +37,6 @@ typedef struct grpc_connectivity_state_watcher {
 typedef struct {
   /** current grpc_connectivity_state */
   gpr_atm current_state_atm;
-  /** error associated with state */
-  grpc_error* current_error;
   /** all our watchers */
   grpc_connectivity_state_watcher* watchers;
   /** a name to help debugging */
@@ -59,7 +57,6 @@ void grpc_connectivity_state_destroy(grpc_connectivity_state_tracker* tracker);
  *  external lock */
 void grpc_connectivity_state_set(grpc_connectivity_state_tracker* tracker,
                                  grpc_connectivity_state state,
-                                 grpc_error* associated_error,
                                  const char* reason);
 
 /** Return true if this connectivity state has watchers.
@@ -71,11 +68,6 @@ bool grpc_connectivity_state_has_watchers(
 grpc_connectivity_state grpc_connectivity_state_check(
     grpc_connectivity_state_tracker* tracker);
 
-/** Return the last seen connectivity state, and the associated error.
-    Access must be serialized with an external lock. */
-grpc_connectivity_state grpc_connectivity_state_get(
-    grpc_connectivity_state_tracker* tracker, grpc_error** error);
-
 /** Return 1 if the channel should start connecting, 0 otherwise.
     If current==NULL cancel notify if it is already queued (success==0 in that
     case).

data/src/core/lib/transport/error_utils.cc

@@ -48,6 +48,18 @@ void grpc_error_get_status(grpc_error* error, grpc_millis deadline,
                            grpc_status_code* code, grpc_slice* slice,
                            grpc_http2_error_code* http_error,
                            const char** error_string) {
+  // Fast path: We expect no error.
+  if (GPR_LIKELY(error == GRPC_ERROR_NONE)) {
+    if (code != nullptr) *code = GRPC_STATUS_OK;
+    if (slice != nullptr) {
+      grpc_error_get_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, slice);
+    }
+    if (http_error != nullptr) {
+      *http_error = GRPC_HTTP2_NO_ERROR;
+    }
+    return;
+  }
+
   // Start with the parent error and recurse through the tree of children
   // until we find the first one that has a status code.
   grpc_error* found_error =

data/src/core/lib/transport/metadata.cc

@@ -41,6 +41,10 @@
 #include "src/core/lib/slice/slice_string_helpers.h"
 #include "src/core/lib/transport/static_metadata.h"
 
+using grpc_core::AllocatedMetadata;
+using grpc_core::InternedMetadata;
+using grpc_core::UserData;
+
 /* There are two kinds of mdelem and mdstr instances.
  * Static instances are declared in static_metadata.{h,c} and
  * are initialized by grpc_mdctx_global_init().
@@ -54,13 +58,40 @@ grpc_core::DebugOnlyTraceFlag grpc_trace_metadata(false, "metadata");
 
 #ifndef NDEBUG
 #define DEBUG_ARGS , const char *file, int line
-#define FWD_DEBUG_ARGS , file, line
-#define REF_MD_LOCKED(shard, s) ref_md_locked((shard), (s), __FILE__, __LINE__)
-#else
+#define FWD_DEBUG_ARGS file, line
+
+void grpc_mdelem_trace_ref(void* md, const grpc_slice& key,
+                           const grpc_slice& value, intptr_t refcnt,
+                           const char* file, int line) {
+  if (grpc_trace_metadata.enabled()) {
+    char* key_str = grpc_slice_to_c_string(key);
+    char* value_str = grpc_slice_to_c_string(value);
+    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+            "ELM   REF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", md, refcnt,
+            refcnt + 1, key_str, value_str);
+    gpr_free(key_str);
+    gpr_free(value_str);
+  }
+}
+
+void grpc_mdelem_trace_unref(void* md, const grpc_slice& key,
+                             const grpc_slice& value, intptr_t refcnt,
+                             const char* file, int line) {
+  if (grpc_trace_metadata.enabled()) {
+    char* key_str = grpc_slice_to_c_string(key);
+    char* value_str = grpc_slice_to_c_string(value);
+    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+            "ELM   UNREF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", md,
+            refcnt, refcnt - 1, key_str, value_str);
+    gpr_free(key_str);
+    gpr_free(value_str);
+  }
+}
+
+#else  // ifndef NDEBUG
 #define DEBUG_ARGS
 #define FWD_DEBUG_ARGS
-#define REF_MD_LOCKED(shard, s) ref_md_locked((shard), (s))
-#endif
+#endif  // ifndef NDEBUG
 
 #define INITIAL_SHARD_CAPACITY 8
 #define LOG2_SHARD_COUNT 4
@@ -69,43 +100,87 @@ grpc_core::DebugOnlyTraceFlag grpc_trace_metadata(false, "metadata");
 #define TABLE_IDX(hash, capacity) (((hash) >> (LOG2_SHARD_COUNT)) % (capacity))
 #define SHARD_IDX(hash) ((hash) & ((1 << (LOG2_SHARD_COUNT)) - 1))
 
-typedef void (*destroy_user_data_func)(void* user_data);
-
-struct UserData {
-  gpr_mu mu_user_data;
-  gpr_atm destroy_user_data;
-  gpr_atm user_data;
-};
-
-/* Shadow structure for grpc_mdelem_data for interned elements */
-typedef struct interned_metadata {
-  /* must be byte compatible with grpc_mdelem_data */
-  grpc_slice key;
-  grpc_slice value;
-
-  /* private only data */
-  gpr_atm refcnt;
-
-  UserData user_data;
+AllocatedMetadata::AllocatedMetadata(const grpc_slice& key,
+                                     const grpc_slice& value)
+    : key_(grpc_slice_ref_internal(key)),
+      value_(grpc_slice_ref_internal(value)),
+      refcnt_(1) {
+#ifndef NDEBUG
+  if (grpc_trace_metadata.enabled()) {
+    char* key_str = grpc_slice_to_c_string(key_);
+    char* value_str = grpc_slice_to_c_string(value_);
+    gpr_log(GPR_DEBUG, "ELM ALLOC:%p:%" PRIdPTR ": '%s' = '%s'", this,
+            RefValue(), key_str, value_str);
+    gpr_free(key_str);
+    gpr_free(value_str);
+  }
+#endif
+}
 
-  struct interned_metadata* bucket_next;
-} interned_metadata;
+AllocatedMetadata::~AllocatedMetadata() {
+  grpc_slice_unref_internal(key_);
+  grpc_slice_unref_internal(value_);
+  void* user_data = user_data_.data.Load(grpc_core::MemoryOrder::RELAXED);
+  if (user_data) {
+    destroy_user_data_func destroy_user_data =
+        user_data_.destroy_user_data.Load(grpc_core::MemoryOrder::RELAXED);
+    destroy_user_data(user_data);
+  }
+}
 
-/* Shadow structure for grpc_mdelem_data for allocated elements */
-typedef struct allocated_metadata {
-  /* must be byte compatible with grpc_mdelem_data */
-  grpc_slice key;
-  grpc_slice value;
+InternedMetadata::InternedMetadata(const grpc_slice& key,
+                                   const grpc_slice& value, uint32_t hash,
+                                   InternedMetadata* next)
+    : key_(grpc_slice_ref_internal(key)),
+      value_(grpc_slice_ref_internal(value)),
+      refcnt_(1),
+      hash_(hash),
+      link_(next) {
+#ifndef NDEBUG
+  if (grpc_trace_metadata.enabled()) {
+    char* key_str = grpc_slice_to_c_string(key_);
+    char* value_str = grpc_slice_to_c_string(value_);
+    gpr_log(GPR_DEBUG, "ELM   NEW:%p:%" PRIdPTR ": '%s' = '%s'", this,
+            RefValue(), key_str, value_str);
+    gpr_free(key_str);
+    gpr_free(value_str);
+  }
+#endif
+}
 
-  /* private only data */
-  gpr_atm refcnt;
+InternedMetadata::~InternedMetadata() {
+  grpc_slice_unref_internal(key_);
+  grpc_slice_unref_internal(value_);
+  void* user_data = user_data_.data.Load(grpc_core::MemoryOrder::RELAXED);
+  if (user_data) {
+    destroy_user_data_func destroy_user_data =
+        user_data_.destroy_user_data.Load(grpc_core::MemoryOrder::RELAXED);
+    destroy_user_data(user_data);
+  }
+}
 
-  UserData user_data;
-} allocated_metadata;
+size_t InternedMetadata::CleanupLinkedMetadata(
+    InternedMetadata::BucketLink* head) {
+  size_t num_freed = 0;
+  InternedMetadata::BucketLink* prev_next = head;
+  InternedMetadata *md, *next;
+
+  for (md = head->next; md; md = next) {
+    next = md->link_.next;
+    if (md->AllRefsDropped()) {
+      prev_next->next = next;
+      grpc_core::Delete(md);
+      num_freed++;
+    } else {
+      prev_next = &md->link_;
+    }
+  }
+  return num_freed;
+}
 
 typedef struct mdtab_shard {
   gpr_mu mu;
-  interned_metadata** elems;
+  InternedMetadata::BucketLink* elems;
   size_t count;
   size_t capacity;
   /** Estimate of the number of unreferenced mdelems in the hash table.
@@ -126,7 +201,7 @@ void grpc_mdctx_global_init(void) {
     shard->count = 0;
     gpr_atm_no_barrier_store(&shard->free_estimate, 0);
     shard->capacity = INITIAL_SHARD_CAPACITY;
-    shard->elems = static_cast<interned_metadata**>(
+    shard->elems = static_cast<InternedMetadata::BucketLink*>(
         gpr_zalloc(sizeof(*shard->elems) * shard->capacity));
   }
 }
@@ -136,7 +211,6 @@ void grpc_mdctx_global_shutdown() {
     mdtab_shard* shard = &g_shards[i];
     gpr_mu_destroy(&shard->mu);
     gc_mdtab(shard);
-    /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */
     if (shard->count != 0) {
       gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata elements were leaked",
               shard->count);
@@ -144,6 +218,7 @@ void grpc_mdctx_global_shutdown() {
         abort();
       }
     }
+    GPR_DEBUG_ASSERT(shard->count == 0);
     gpr_free(shard->elems);
   }
 }
@@ -154,57 +229,34 @@ static int is_mdelem_static(grpc_mdelem e) {
              &grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
 }
 
-static void ref_md_locked(mdtab_shard* shard,
-                          interned_metadata* md DEBUG_ARGS) {
+void InternedMetadata::RefWithShardLocked(mdtab_shard* shard) {
 #ifndef NDEBUG
   if (grpc_trace_metadata.enabled()) {
-    char* key_str = grpc_slice_to_c_string(md->key);
-    char* value_str = grpc_slice_to_c_string(md->value);
-    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "ELM   REF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", (void*)md,
-            gpr_atm_no_barrier_load(&md->refcnt),
-            gpr_atm_no_barrier_load(&md->refcnt) + 1, key_str, value_str);
+    char* key_str = grpc_slice_to_c_string(key_);
+    char* value_str = grpc_slice_to_c_string(value_);
+    intptr_t value = RefValue();
+    gpr_log(__FILE__, __LINE__, GPR_LOG_SEVERITY_DEBUG,
+            "ELM   REF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", this, value,
+            value + 1, key_str, value_str);
     gpr_free(key_str);
     gpr_free(value_str);
   }
 #endif
-  if (0 == gpr_atm_no_barrier_fetch_add(&md->refcnt, 1)) {
+  if (FirstRef()) {
     gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -1);
   }
 }
 
 static void gc_mdtab(mdtab_shard* shard) {
   GPR_TIMER_SCOPE("gc_mdtab", 0);
-
-  size_t i;
-  interned_metadata** prev_next;
-  interned_metadata *md, *next;
-  gpr_atm num_freed = 0;
-
-  for (i = 0; i < shard->capacity; i++) {
-    prev_next = &shard->elems[i];
-    for (md = shard->elems[i]; md; md = next) {
-      void* user_data =
-          (void*)gpr_atm_no_barrier_load(&md->user_data.user_data);
-      next = md->bucket_next;
-      if (gpr_atm_acq_load(&md->refcnt) == 0) {
-        grpc_slice_unref_internal(md->key);
-        grpc_slice_unref_internal(md->value);
-        if (md->user_data.user_data) {
-          ((destroy_user_data_func)gpr_atm_no_barrier_load(
-              &md->user_data.destroy_user_data))(user_data);
-        }
-        gpr_mu_destroy(&md->user_data.mu_user_data);
-        gpr_free(md);
-        *prev_next = next;
-        num_freed++;
-        shard->count--;
-      } else {
-        prev_next = &md->bucket_next;
-      }
-    }
+  size_t num_freed = 0;
+  for (size_t i = 0; i < shard->capacity; ++i) {
+    intptr_t freed = InternedMetadata::CleanupLinkedMetadata(&shard->elems[i]);
+    num_freed += freed;
+    shard->count -= freed;
   }
-  gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -num_freed);
+  gpr_atm_no_barrier_fetch_add(&shard->free_estimate,
+                               -static_cast<intptr_t>(num_freed));
 }
 
 static void grow_mdtab(mdtab_shard* shard) {
@@ -212,22 +264,21 @@ static void grow_mdtab(mdtab_shard* shard) {
 
   size_t capacity = shard->capacity * 2;
   size_t i;
-  interned_metadata** mdtab;
-  interned_metadata *md, *next;
+  InternedMetadata::BucketLink* mdtab;
+  InternedMetadata *md, *next;
   uint32_t hash;
 
-  mdtab = static_cast<interned_metadata**>(
-      gpr_zalloc(sizeof(interned_metadata*) * capacity));
+  mdtab = static_cast<InternedMetadata::BucketLink*>(
+      gpr_zalloc(sizeof(InternedMetadata::BucketLink) * capacity));
 
   for (i = 0; i < shard->capacity; i++) {
-    for (md = shard->elems[i]; md; md = next) {
+    for (md = shard->elems[i].next; md; md = next) {
       size_t idx;
-      hash = GRPC_MDSTR_KV_HASH(grpc_slice_hash(md->key),
-                                grpc_slice_hash(md->value));
-      next = md->bucket_next;
+      hash = md->hash();
+      next = md->bucket_next();
       idx = TABLE_IDX(hash, capacity);
-      md->bucket_next = mdtab[idx];
-      mdtab[idx] = md;
+      md->set_bucket_next(mdtab[idx].next);
+      mdtab[idx].next = md;
     }
   }
   gpr_free(shard->elems);
@@ -247,34 +298,22 @@ static void rehash_mdtab(mdtab_shard* shard) {
 grpc_mdelem grpc_mdelem_create(
     const grpc_slice& key, const grpc_slice& value,
     grpc_mdelem_data* compatible_external_backing_store) {
+  // External storage if either slice is not interned and the caller already
+  // created a backing store. If no backing store, we allocate one.
   if (!grpc_slice_is_interned(key) || !grpc_slice_is_interned(value)) {
     if (compatible_external_backing_store != nullptr) {
+      // Caller provided backing store.
       return GRPC_MAKE_MDELEM(compatible_external_backing_store,
                               GRPC_MDELEM_STORAGE_EXTERNAL);
+    } else {
+      // We allocate backing store.
+      return GRPC_MAKE_MDELEM(grpc_core::New<AllocatedMetadata>(key, value),
+                              GRPC_MDELEM_STORAGE_ALLOCATED);
     }
-
-    allocated_metadata* allocated =
-        static_cast<allocated_metadata*>(gpr_malloc(sizeof(*allocated)));
-    allocated->key = grpc_slice_ref_internal(key);
-    allocated->value = grpc_slice_ref_internal(value);
-    gpr_atm_rel_store(&allocated->refcnt, 1);
-    allocated->user_data.user_data = 0;
-    allocated->user_data.destroy_user_data = 0;
-    gpr_mu_init(&allocated->user_data.mu_user_data);
-#ifndef NDEBUG
-    if (grpc_trace_metadata.enabled()) {
-      char* key_str = grpc_slice_to_c_string(allocated->key);
-      char* value_str = grpc_slice_to_c_string(allocated->value);
-      gpr_log(GPR_DEBUG, "ELM ALLOC:%p:%" PRIdPTR ": '%s' = '%s'",
-              (void*)allocated, gpr_atm_no_barrier_load(&allocated->refcnt),
-              key_str, value_str);
-      gpr_free(key_str);
-      gpr_free(value_str);
-    }
-#endif
-    return GRPC_MAKE_MDELEM(allocated, GRPC_MDELEM_STORAGE_ALLOCATED);
   }
 
+  // Not all static slice input yields a statically stored metadata element.
+  // It may be worth documenting why.
   if (GRPC_IS_STATIC_METADATA_STRING(key) &&
       GRPC_IS_STATIC_METADATA_STRING(value)) {
     grpc_mdelem static_elem = grpc_static_mdelem_for_static_strings(
@@ -286,7 +325,7 @@ grpc_mdelem grpc_mdelem_create(
 
   uint32_t hash =
       GRPC_MDSTR_KV_HASH(grpc_slice_hash(key), grpc_slice_hash(value));
-  interned_metadata* md;
+  InternedMetadata* md;
   mdtab_shard* shard = &g_shards[SHARD_IDX(hash)];
   size_t idx;
 
@@ -296,34 +335,18 @@ grpc_mdelem grpc_mdelem_create(
 
   idx = TABLE_IDX(hash, shard->capacity);
   /* search for an existing pair */
-  for (md = shard->elems[idx]; md; md = md->bucket_next) {
-    if (grpc_slice_eq(key, md->key) && grpc_slice_eq(value, md->value)) {
-      REF_MD_LOCKED(shard, md);
+  for (md = shard->elems[idx].next; md; md = md->bucket_next()) {
+    if (grpc_slice_eq(key, md->key()) && grpc_slice_eq(value, md->value())) {
+      md->RefWithShardLocked(shard);
       gpr_mu_unlock(&shard->mu);
       return GRPC_MAKE_MDELEM(md, GRPC_MDELEM_STORAGE_INTERNED);
     }
   }
 
   /* not found: create a new pair */
-  md = static_cast<interned_metadata*>(gpr_malloc(sizeof(interned_metadata)));
-  gpr_atm_rel_store(&md->refcnt, 1);
-  md->key = grpc_slice_ref_internal(key);
-  md->value = grpc_slice_ref_internal(value);
-  md->user_data.user_data = 0;
-  md->user_data.destroy_user_data = 0;
-  md->bucket_next = shard->elems[idx];
-  shard->elems[idx] = md;
-  gpr_mu_init(&md->user_data.mu_user_data);
-#ifndef NDEBUG
-  if (grpc_trace_metadata.enabled()) {
-    char* key_str = grpc_slice_to_c_string(md->key);
-    char* value_str = grpc_slice_to_c_string(md->value);
-    gpr_log(GPR_DEBUG, "ELM   NEW:%p:%" PRIdPTR ": '%s' = '%s'", (void*)md,
-            gpr_atm_no_barrier_load(&md->refcnt), key_str, value_str);
-    gpr_free(key_str);
-    gpr_free(value_str);
-  }
-#endif
+  md = grpc_core::New<InternedMetadata>(key, value, hash,
+                                        shard->elems[idx].next);
+  shard->elems[idx].next = md;
   shard->count++;
 
   if (shard->count > shard->capacity * 2) {
@@ -354,130 +377,10 @@ grpc_mdelem grpc_mdelem_from_grpc_metadata(grpc_metadata* metadata) {
       changed ? nullptr : reinterpret_cast<grpc_mdelem_data*>(metadata));
 }
 
-grpc_mdelem grpc_mdelem_ref(grpc_mdelem gmd DEBUG_ARGS) {
-  switch (GRPC_MDELEM_STORAGE(gmd)) {
-    case GRPC_MDELEM_STORAGE_EXTERNAL:
-    case GRPC_MDELEM_STORAGE_STATIC:
-      break;
-    case GRPC_MDELEM_STORAGE_INTERNED: {
-      interned_metadata* md =
-          reinterpret_cast<interned_metadata*> GRPC_MDELEM_DATA(gmd);
-#ifndef NDEBUG
-      if (grpc_trace_metadata.enabled()) {
-        char* key_str = grpc_slice_to_c_string(md->key);
-        char* value_str = grpc_slice_to_c_string(md->value);
-        gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "ELM   REF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'",
-                (void*)md, gpr_atm_no_barrier_load(&md->refcnt),
-                gpr_atm_no_barrier_load(&md->refcnt) + 1, key_str, value_str);
-        gpr_free(key_str);
-        gpr_free(value_str);
-      }
-#endif
-      /* we can assume the ref count is >= 1 as the application is calling
-         this function - meaning that no adjustment to mdtab_free is necessary,
-         simplifying the logic here to be just an atomic increment */
-      /* use C assert to have this removed in opt builds */
-      GPR_ASSERT(gpr_atm_no_barrier_load(&md->refcnt) >= 1);
-      gpr_atm_no_barrier_fetch_add(&md->refcnt, 1);
-      break;
-    }
-    case GRPC_MDELEM_STORAGE_ALLOCATED: {
-      allocated_metadata* md =
-          reinterpret_cast<allocated_metadata*> GRPC_MDELEM_DATA(gmd);
-#ifndef NDEBUG
-      if (grpc_trace_metadata.enabled()) {
-        char* key_str = grpc_slice_to_c_string(md->key);
-        char* value_str = grpc_slice_to_c_string(md->value);
-        gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "ELM   REF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'",
-                (void*)md, gpr_atm_no_barrier_load(&md->refcnt),
-                gpr_atm_no_barrier_load(&md->refcnt) + 1, key_str, value_str);
-        gpr_free(key_str);
-        gpr_free(value_str);
-      }
-#endif
-      /* we can assume the ref count is >= 1 as the application is calling
-         this function - meaning that no adjustment to mdtab_free is necessary,
-         simplifying the logic here to be just an atomic increment */
-      /* use C assert to have this removed in opt builds */
-      gpr_atm_no_barrier_fetch_add(&md->refcnt, 1);
-      break;
-    }
-  }
-  return gmd;
-}
-
-void grpc_mdelem_unref(grpc_mdelem gmd DEBUG_ARGS) {
-  switch (GRPC_MDELEM_STORAGE(gmd)) {
-    case GRPC_MDELEM_STORAGE_EXTERNAL:
-    case GRPC_MDELEM_STORAGE_STATIC:
-      break;
-    case GRPC_MDELEM_STORAGE_INTERNED: {
-      interned_metadata* md =
-          reinterpret_cast<interned_metadata*> GRPC_MDELEM_DATA(gmd);
-#ifndef NDEBUG
-      if (grpc_trace_metadata.enabled()) {
-        char* key_str = grpc_slice_to_c_string(md->key);
-        char* value_str = grpc_slice_to_c_string(md->value);
-        gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "ELM UNREF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'",
-                (void*)md, gpr_atm_no_barrier_load(&md->refcnt),
-                gpr_atm_no_barrier_load(&md->refcnt) - 1, key_str, value_str);
-        gpr_free(key_str);
-        gpr_free(value_str);
-      }
-#endif
-      uint32_t hash = GRPC_MDSTR_KV_HASH(grpc_slice_hash(md->key),
-                                         grpc_slice_hash(md->value));
-      const gpr_atm prev_refcount = gpr_atm_full_fetch_add(&md->refcnt, -1);
-      GPR_ASSERT(prev_refcount >= 1);
-      if (1 == prev_refcount) {
-        /* once the refcount hits zero, some other thread can come along and
-           free md at any time: it's unsafe from this point on to access it */
-        mdtab_shard* shard = &g_shards[SHARD_IDX(hash)];
-        gpr_atm_no_barrier_fetch_add(&shard->free_estimate, 1);
-      }
-      break;
-    }
-    case GRPC_MDELEM_STORAGE_ALLOCATED: {
-      allocated_metadata* md =
-          reinterpret_cast<allocated_metadata*> GRPC_MDELEM_DATA(gmd);
-#ifndef NDEBUG
-      if (grpc_trace_metadata.enabled()) {
-        char* key_str = grpc_slice_to_c_string(md->key);
-        char* value_str = grpc_slice_to_c_string(md->value);
-        gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-                "ELM UNREF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'",
-                (void*)md, gpr_atm_no_barrier_load(&md->refcnt),
-                gpr_atm_no_barrier_load(&md->refcnt) - 1, key_str, value_str);
-        gpr_free(key_str);
-        gpr_free(value_str);
-      }
-#endif
-      const gpr_atm prev_refcount = gpr_atm_full_fetch_add(&md->refcnt, -1);
-      GPR_ASSERT(prev_refcount >= 1);
-      if (1 == prev_refcount) {
-        grpc_slice_unref_internal(md->key);
-        grpc_slice_unref_internal(md->value);
-        if (md->user_data.user_data) {
-          destroy_user_data_func destroy_user_data =
-              (destroy_user_data_func)gpr_atm_no_barrier_load(
-                  &md->user_data.destroy_user_data);
-          destroy_user_data((void*)md->user_data.user_data);
-        }
-        gpr_mu_destroy(&md->user_data.mu_user_data);
-        gpr_free(md);
-      }
-      break;
-    }
-  }
-}
-
 static void* get_user_data(UserData* user_data, void (*destroy_func)(void*)) {
-  if (gpr_atm_acq_load(&user_data->destroy_user_data) ==
-      (gpr_atm)destroy_func) {
-    return (void*)gpr_atm_no_barrier_load(&user_data->user_data);
+  if (user_data->destroy_user_data.Load(grpc_core::MemoryOrder::ACQUIRE) ==
+      destroy_func) {
+    return user_data->data.Load(grpc_core::MemoryOrder::RELAXED);
   } else {
     return nullptr;
   }
@@ -491,57 +394,52 @@ void* grpc_mdelem_get_user_data(grpc_mdelem md, void (*destroy_func)(void*)) {
       return (void*)grpc_static_mdelem_user_data[GRPC_MDELEM_DATA(md) -
                                                  grpc_static_mdelem_table];
     case GRPC_MDELEM_STORAGE_ALLOCATED: {
-      allocated_metadata* am =
-          reinterpret_cast<allocated_metadata*>(GRPC_MDELEM_DATA(md));
-      return get_user_data(&am->user_data, destroy_func);
+      auto* am = reinterpret_cast<AllocatedMetadata*>(GRPC_MDELEM_DATA(md));
+      return get_user_data(am->user_data(), destroy_func);
     }
     case GRPC_MDELEM_STORAGE_INTERNED: {
-      interned_metadata* im =
-          reinterpret_cast<interned_metadata*> GRPC_MDELEM_DATA(md);
-      return get_user_data(&im->user_data, destroy_func);
+      auto* im = reinterpret_cast<InternedMetadata*> GRPC_MDELEM_DATA(md);
+      return get_user_data(im->user_data(), destroy_func);
     }
   }
   GPR_UNREACHABLE_CODE(return nullptr);
 }
 
 static void* set_user_data(UserData* ud, void (*destroy_func)(void*),
-                           void* user_data) {
-  GPR_ASSERT((user_data == nullptr) == (destroy_func == nullptr));
-  gpr_mu_lock(&ud->mu_user_data);
-  if (gpr_atm_no_barrier_load(&ud->destroy_user_data)) {
+                           void* data) {
+  GPR_ASSERT((data == nullptr) == (destroy_func == nullptr));
+  grpc_core::ReleasableMutexLock lock(&ud->mu_user_data);
+  if (ud->destroy_user_data.Load(grpc_core::MemoryOrder::RELAXED)) {
     /* user data can only be set once */
-    gpr_mu_unlock(&ud->mu_user_data);
+    lock.Unlock();
     if (destroy_func != nullptr) {
-      destroy_func(user_data);
+      destroy_func(data);
     }
-    return (void*)gpr_atm_no_barrier_load(&ud->user_data);
+    return ud->data.Load(grpc_core::MemoryOrder::RELAXED);
   }
-  gpr_atm_no_barrier_store(&ud->user_data, (gpr_atm)user_data);
-  gpr_atm_rel_store(&ud->destroy_user_data, (gpr_atm)destroy_func);
-  gpr_mu_unlock(&ud->mu_user_data);
-  return user_data;
+  ud->data.Store(data, grpc_core::MemoryOrder::RELAXED);
+  ud->destroy_user_data.Store(destroy_func, grpc_core::MemoryOrder::RELEASE);
+  return data;
 }
 
 void* grpc_mdelem_set_user_data(grpc_mdelem md, void (*destroy_func)(void*),
-                                void* user_data) {
+                                void* data) {
   switch (GRPC_MDELEM_STORAGE(md)) {
     case GRPC_MDELEM_STORAGE_EXTERNAL:
-      destroy_func(user_data);
+      destroy_func(data);
       return nullptr;
     case GRPC_MDELEM_STORAGE_STATIC:
-      destroy_func(user_data);
+      destroy_func(data);
       return (void*)grpc_static_mdelem_user_data[GRPC_MDELEM_DATA(md) -
                                                  grpc_static_mdelem_table];
     case GRPC_MDELEM_STORAGE_ALLOCATED: {
-      allocated_metadata* am =
-          reinterpret_cast<allocated_metadata*>(GRPC_MDELEM_DATA(md));
-      return set_user_data(&am->user_data, destroy_func, user_data);
+      auto* am = reinterpret_cast<AllocatedMetadata*>(GRPC_MDELEM_DATA(md));
+      return set_user_data(am->user_data(), destroy_func, data);
     }
     case GRPC_MDELEM_STORAGE_INTERNED: {
-      interned_metadata* im =
-          reinterpret_cast<interned_metadata*> GRPC_MDELEM_DATA(md);
+      auto* im = reinterpret_cast<InternedMetadata*> GRPC_MDELEM_DATA(md);
       GPR_ASSERT(!is_mdelem_static(md));
-      return set_user_data(&im->user_data, destroy_func, user_data);
+      return set_user_data(im->user_data(), destroy_func, data);
     }
   }
   GPR_UNREACHABLE_CODE(return nullptr);
@@ -554,3 +452,33 @@ bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b) {
   return grpc_slice_eq(GRPC_MDKEY(a), GRPC_MDKEY(b)) &&
          grpc_slice_eq(GRPC_MDVALUE(a), GRPC_MDVALUE(b));
 }
+
+static void note_disposed_interned_metadata(uint32_t hash) {
+  mdtab_shard* shard = &g_shards[SHARD_IDX(hash)];
+  gpr_atm_no_barrier_fetch_add(&shard->free_estimate, 1);
+}
+
+void grpc_mdelem_do_unref(grpc_mdelem gmd DEBUG_ARGS) {
+  switch (GRPC_MDELEM_STORAGE(gmd)) {
+    case GRPC_MDELEM_STORAGE_EXTERNAL:
+    case GRPC_MDELEM_STORAGE_STATIC:
+      return;
+    case GRPC_MDELEM_STORAGE_INTERNED: {
+      auto* md = reinterpret_cast<InternedMetadata*> GRPC_MDELEM_DATA(gmd);
+      uint32_t hash = md->hash();
+      if (GPR_UNLIKELY(md->Unref(FWD_DEBUG_ARGS))) {
+        /* once the refcount hits zero, some other thread can come along and
+           free md at any time: it's unsafe from this point on to access it */
+        note_disposed_interned_metadata(hash);
+      }
+      break;
+    }
+    case GRPC_MDELEM_STORAGE_ALLOCATED: {
+      auto* md = reinterpret_cast<AllocatedMetadata*> GRPC_MDELEM_DATA(gmd);
+      if (GPR_UNLIKELY(md->Unref(FWD_DEBUG_ARGS))) {
+        grpc_core::Delete(md);
+      }
+      break;
+    }
+  }
+}