wearefair-grpc 1.3.1.pre.a

data/src/core/lib/transport/error_utils.c

@@ -0,0 +1,124 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/transport/error_utils.h"
+
+#include "src/core/lib/iomgr/error_internal.h"
+#include "src/core/lib/transport/status_conversion.h"
+
+static grpc_error *recursively_find_error_with_field(grpc_error *error,
+                                                     grpc_error_ints which) {
+  // If the error itself has a status code, return it.
+  if (grpc_error_get_int(error, which, NULL)) {
+    return error;
+  }
+  if (grpc_error_is_special(error)) return NULL;
+  // Otherwise, search through its children.
+  uint8_t slot = error->first_err;
+  while (slot != UINT8_MAX) {
+    grpc_linked_error *lerr = (grpc_linked_error *)(error->arena + slot);
+    grpc_error *result = recursively_find_error_with_field(lerr->err, which);
+    if (result) return result;
+    slot = lerr->next;
+  }
+  return NULL;
+}
+
+void grpc_error_get_status(grpc_error *error, gpr_timespec deadline,
+                           grpc_status_code *code, grpc_slice *slice,
+                           grpc_http2_error_code *http_error) {
+  // Start with the parent error and recurse through the tree of children
+  // until we find the first one that has a status code.
+  grpc_error *found_error =
+      recursively_find_error_with_field(error, GRPC_ERROR_INT_GRPC_STATUS);
+  if (found_error == NULL) {
+    /// If no grpc-status exists, retry through the tree to find a http2 error
+    /// code
+    found_error =
+        recursively_find_error_with_field(error, GRPC_ERROR_INT_HTTP2_ERROR);
+  }
+
+  // If we found an error with a status code above, use that; otherwise,
+  // fall back to using the parent error.
+  if (found_error == NULL) found_error = error;
+
+  grpc_status_code status = GRPC_STATUS_UNKNOWN;
+  intptr_t integer;
+  if (grpc_error_get_int(found_error, GRPC_ERROR_INT_GRPC_STATUS, &integer)) {
+    status = (grpc_status_code)integer;
+  } else if (grpc_error_get_int(found_error, GRPC_ERROR_INT_HTTP2_ERROR,
+                                &integer)) {
+    status = grpc_http2_error_to_grpc_status((grpc_http2_error_code)integer,
+                                             deadline);
+  }
+  if (code != NULL) *code = status;
+
+  if (http_error != NULL) {
+    if (grpc_error_get_int(found_error, GRPC_ERROR_INT_HTTP2_ERROR, &integer)) {
+      *http_error = (grpc_http2_error_code)integer;
+    } else if (grpc_error_get_int(found_error, GRPC_ERROR_INT_GRPC_STATUS,
+                                  &integer)) {
+      *http_error = grpc_status_to_http2_error((grpc_status_code)integer);
+    } else {
+      *http_error = found_error == GRPC_ERROR_NONE ? GRPC_HTTP2_NO_ERROR
+                                                   : GRPC_HTTP2_INTERNAL_ERROR;
+    }
+  }
+
+  // If the error has a status message, use it.  Otherwise, fall back to
+  // the error description.
+  if (slice != NULL) {
+    if (!grpc_error_get_str(found_error, GRPC_ERROR_STR_GRPC_MESSAGE, slice)) {
+      if (!grpc_error_get_str(found_error, GRPC_ERROR_STR_DESCRIPTION, slice)) {
+        *slice = grpc_slice_from_static_string("unknown error");
+      }
+    }
+  }
+
+  if (found_error == NULL) found_error = error;
+}
+
+bool grpc_error_has_clear_grpc_status(grpc_error *error) {
+  if (grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, NULL)) {
+    return true;
+  }
+  uint8_t slot = error->first_err;
+  while (slot != UINT8_MAX) {
+    grpc_linked_error *lerr = (grpc_linked_error *)(error->arena + slot);
+    if (grpc_error_has_clear_grpc_status(lerr->err)) {
+      return true;
+    }
+    slot = lerr->next;
+  }
+  return false;
+}

data/src/core/lib/transport/error_utils.h

@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_TRANSPORT_ERROR_UTILS_H
+#define GRPC_CORE_LIB_TRANSPORT_ERROR_UTILS_H
+
+#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/transport/http2_errors.h"
+
+/// A utility function to get the status code and message to be returned
+/// to the application.  If not set in the top-level message, looks
+/// through child errors until it finds the first one with these attributes.
+/// All attributes are pulled from the same child error. If any of the
+/// attributes (code, msg, http_status) are unneeded, they can be passed as
+/// NULL.
+void grpc_error_get_status(grpc_error *error, gpr_timespec deadline,
+                           grpc_status_code *code, grpc_slice *slice,
+                           grpc_http2_error_code *http_status);
+
+/// A utility function to check whether there is a clear status code that
+/// doesn't need to be guessed in \a error. This means that \a error or some
+/// child has GRPC_ERROR_INT_GRPC_STATUS set, or that it is GRPC_ERROR_NONE or
+/// GRPC_ERROR_CANCELLED
+bool grpc_error_has_clear_grpc_status(grpc_error *error);
+
+#endif /* GRPC_CORE_LIB_TRANSPORT_ERROR_UTILS_H */

data/src/core/lib/transport/http2_errors.h

@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_TRANSPORT_HTTP2_ERRORS_H
+#define GRPC_CORE_LIB_TRANSPORT_HTTP2_ERRORS_H
+
+/* error codes for RST_STREAM from http2 draft 14 section 7 */
+typedef enum {
+  GRPC_HTTP2_NO_ERROR = 0x0,
+  GRPC_HTTP2_PROTOCOL_ERROR = 0x1,
+  GRPC_HTTP2_INTERNAL_ERROR = 0x2,
+  GRPC_HTTP2_FLOW_CONTROL_ERROR = 0x3,
+  GRPC_HTTP2_SETTINGS_TIMEOUT = 0x4,
+  GRPC_HTTP2_STREAM_CLOSED = 0x5,
+  GRPC_HTTP2_FRAME_SIZE_ERROR = 0x6,
+  GRPC_HTTP2_REFUSED_STREAM = 0x7,
+  GRPC_HTTP2_CANCEL = 0x8,
+  GRPC_HTTP2_COMPRESSION_ERROR = 0x9,
+  GRPC_HTTP2_CONNECT_ERROR = 0xa,
+  GRPC_HTTP2_ENHANCE_YOUR_CALM = 0xb,
+  GRPC_HTTP2_INADEQUATE_SECURITY = 0xc,
+  /* force use of a default clause */
+  GRPC_HTTP2__ERROR_DO_NOT_USE = -1
+} grpc_http2_error_code;
+
+#endif /* GRPC_CORE_LIB_TRANSPORT_HTTP2_ERRORS_H */

data/src/core/lib/transport/metadata.c

@@ -0,0 +1,529 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/transport/metadata.h"
+
+#include <assert.h>
+#include <stddef.h>
+#include <string.h>
+
+#include <grpc/compression.h>
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/atm.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/time.h>
+
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/slice/slice_string_helpers.h"
+#include "src/core/lib/support/murmur_hash.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/transport/static_metadata.h"
+
+/* There are two kinds of mdelem and mdstr instances.
+ * Static instances are declared in static_metadata.{h,c} and
+ * are initialized by grpc_mdctx_global_init().
+ * Dynamic instances are stored in hash tables on grpc_mdctx, and are backed
+ * by internal_string and internal_element structures.
+ * Internal helper functions here-in (is_mdstr_static, is_mdelem_static) are
+ * used to determine which kind of element a pointer refers to.
+ */
+
+#ifdef GRPC_METADATA_REFCOUNT_DEBUG
+#define DEBUG_ARGS , const char *file, int line
+#define FWD_DEBUG_ARGS , file, line
+#define REF_MD_LOCKED(shard, s) ref_md_locked((shard), (s), __FILE__, __LINE__)
+#else
+#define DEBUG_ARGS
+#define FWD_DEBUG_ARGS
+#define REF_MD_LOCKED(shard, s) ref_md_locked((shard), (s))
+#endif
+
+#define INITIAL_SHARD_CAPACITY 8
+#define LOG2_SHARD_COUNT 4
+#define SHARD_COUNT ((size_t)(1 << LOG2_SHARD_COUNT))
+
+#define TABLE_IDX(hash, capacity) (((hash) >> (LOG2_SHARD_COUNT)) % (capacity))
+#define SHARD_IDX(hash) ((hash) & ((1 << (LOG2_SHARD_COUNT)) - 1))
+
+typedef void (*destroy_user_data_func)(void *user_data);
+
+/* Shadow structure for grpc_mdelem_data for interned elements */
+typedef struct interned_metadata {
+  /* must be byte compatible with grpc_mdelem_data */
+  grpc_slice key;
+  grpc_slice value;
+
+  /* private only data */
+  gpr_atm refcnt;
+
+  gpr_mu mu_user_data;
+  gpr_atm destroy_user_data;
+  gpr_atm user_data;
+
+  struct interned_metadata *bucket_next;
+} interned_metadata;
+
+/* Shadow structure for grpc_mdelem_data for allocated elements */
+typedef struct allocated_metadata {
+  /* must be byte compatible with grpc_mdelem_data */
+  grpc_slice key;
+  grpc_slice value;
+
+  /* private only data */
+  gpr_atm refcnt;
+} allocated_metadata;
+
+typedef struct mdtab_shard {
+  gpr_mu mu;
+  interned_metadata **elems;
+  size_t count;
+  size_t capacity;
+  /** Estimate of the number of unreferenced mdelems in the hash table.
+      This will eventually converge to the exact number, but it's instantaneous
+      accuracy is not guaranteed */
+  gpr_atm free_estimate;
+} mdtab_shard;
+
+static mdtab_shard g_shards[SHARD_COUNT];
+
+static void gc_mdtab(grpc_exec_ctx *exec_ctx, mdtab_shard *shard);
+
+void grpc_mdctx_global_init(void) {
+  /* initialize shards */
+  for (size_t i = 0; i < SHARD_COUNT; i++) {
+    mdtab_shard *shard = &g_shards[i];
+    gpr_mu_init(&shard->mu);
+    shard->count = 0;
+    gpr_atm_no_barrier_store(&shard->free_estimate, 0);
+    shard->capacity = INITIAL_SHARD_CAPACITY;
+    shard->elems = gpr_zalloc(sizeof(*shard->elems) * shard->capacity);
+  }
+}
+
+void grpc_mdctx_global_shutdown(grpc_exec_ctx *exec_ctx) {
+  for (size_t i = 0; i < SHARD_COUNT; i++) {
+    mdtab_shard *shard = &g_shards[i];
+    gpr_mu_destroy(&shard->mu);
+    gc_mdtab(exec_ctx, shard);
+    /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */
+    if (shard->count != 0) {
+      gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata elements were leaked",
+              shard->count);
+      if (grpc_iomgr_abort_on_leaks()) {
+        abort();
+      }
+    }
+    gpr_free(shard->elems);
+  }
+}
+
+static int is_mdelem_static(grpc_mdelem e) {
+  return GRPC_MDELEM_DATA(e) >= &grpc_static_mdelem_table[0] &&
+         GRPC_MDELEM_DATA(e) <
+             &grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
+}
+
+static void ref_md_locked(mdtab_shard *shard,
+                          interned_metadata *md DEBUG_ARGS) {
+#ifdef GRPC_METADATA_REFCOUNT_DEBUG
+  char *key_str = grpc_slice_to_c_string(md->key);
+  char *value_str = grpc_slice_to_c_string(md->value);
+  gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+          "ELM   REF:%p:%zu->%zu: '%s' = '%s'", (void *)md,
+          gpr_atm_no_barrier_load(&md->refcnt),
+          gpr_atm_no_barrier_load(&md->refcnt) + 1, key_str, value_str);
+  gpr_free(key_str);
+  gpr_free(value_str);
+#endif
+  if (0 == gpr_atm_no_barrier_fetch_add(&md->refcnt, 1)) {
+    gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -1);
+  }
+}
+
+static void gc_mdtab(grpc_exec_ctx *exec_ctx, mdtab_shard *shard) {
+  size_t i;
+  interned_metadata **prev_next;
+  interned_metadata *md, *next;
+  gpr_atm num_freed = 0;
+
+  GPR_TIMER_BEGIN("gc_mdtab", 0);
+  for (i = 0; i < shard->capacity; i++) {
+    prev_next = &shard->elems[i];
+    for (md = shard->elems[i]; md; md = next) {
+      void *user_data = (void *)gpr_atm_no_barrier_load(&md->user_data);
+      next = md->bucket_next;
+      if (gpr_atm_acq_load(&md->refcnt) == 0) {
+        grpc_slice_unref_internal(exec_ctx, md->key);
+        grpc_slice_unref_internal(exec_ctx, md->value);
+        if (md->user_data) {
+          ((destroy_user_data_func)gpr_atm_no_barrier_load(
+              &md->destroy_user_data))(user_data);
+        }
+        gpr_free(md);
+        *prev_next = next;
+        num_freed++;
+        shard->count--;
+      } else {
+        prev_next = &md->bucket_next;
+      }
+    }
+  }
+  gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -num_freed);
+  GPR_TIMER_END("gc_mdtab", 0);
+}
+
+static void grow_mdtab(mdtab_shard *shard) {
+  size_t capacity = shard->capacity * 2;
+  size_t i;
+  interned_metadata **mdtab;
+  interned_metadata *md, *next;
+  uint32_t hash;
+
+  GPR_TIMER_BEGIN("grow_mdtab", 0);
+
+  mdtab = gpr_zalloc(sizeof(interned_metadata *) * capacity);
+
+  for (i = 0; i < shard->capacity; i++) {
+    for (md = shard->elems[i]; md; md = next) {
+      size_t idx;
+      hash = GRPC_MDSTR_KV_HASH(grpc_slice_hash(md->key),
+                                grpc_slice_hash(md->value));
+      next = md->bucket_next;
+      idx = TABLE_IDX(hash, capacity);
+      md->bucket_next = mdtab[idx];
+      mdtab[idx] = md;
+    }
+  }
+
+  gpr_free(shard->elems);
+  shard->elems = mdtab;
+  shard->capacity = capacity;
+
+  GPR_TIMER_END("grow_mdtab", 0);
+}
+
+static void rehash_mdtab(grpc_exec_ctx *exec_ctx, mdtab_shard *shard) {
+  if (gpr_atm_no_barrier_load(&shard->free_estimate) >
+      (gpr_atm)(shard->capacity / 4)) {
+    gc_mdtab(exec_ctx, shard);
+  } else {
+    grow_mdtab(shard);
+  }
+}
+
+grpc_mdelem grpc_mdelem_create(
+    grpc_exec_ctx *exec_ctx, grpc_slice key, grpc_slice value,
+    grpc_mdelem_data *compatible_external_backing_store) {
+  if (!grpc_slice_is_interned(key) || !grpc_slice_is_interned(value)) {
+    if (compatible_external_backing_store != NULL) {
+      return GRPC_MAKE_MDELEM(compatible_external_backing_store,
+                              GRPC_MDELEM_STORAGE_EXTERNAL);
+    }
+
+    allocated_metadata *allocated = gpr_malloc(sizeof(*allocated));
+    allocated->key = grpc_slice_ref_internal(key);
+    allocated->value = grpc_slice_ref_internal(value);
+    gpr_atm_rel_store(&allocated->refcnt, 1);
+#ifdef GRPC_METADATA_REFCOUNT_DEBUG
+    char *key_str = grpc_slice_to_c_string(allocated->key);
+    char *value_str = grpc_slice_to_c_string(allocated->value);
+    gpr_log(GPR_DEBUG, "ELM ALLOC:%p:%zu: '%s' = '%s'", (void *)allocated,
+            gpr_atm_no_barrier_load(&allocated->refcnt), key_str, value_str);
+    gpr_free(key_str);
+    gpr_free(value_str);
+#endif
+    return GRPC_MAKE_MDELEM(allocated, GRPC_MDELEM_STORAGE_ALLOCATED);
+  }
+
+  if (GRPC_IS_STATIC_METADATA_STRING(key) &&
+      GRPC_IS_STATIC_METADATA_STRING(value)) {
+    grpc_mdelem static_elem = grpc_static_mdelem_for_static_strings(
+        GRPC_STATIC_METADATA_INDEX(key), GRPC_STATIC_METADATA_INDEX(value));
+    if (!GRPC_MDISNULL(static_elem)) {
+      return static_elem;
+    }
+  }
+
+  uint32_t hash =
+      GRPC_MDSTR_KV_HASH(grpc_slice_hash(key), grpc_slice_hash(value));
+  interned_metadata *md;
+  mdtab_shard *shard = &g_shards[SHARD_IDX(hash)];
+  size_t idx;
+
+  GPR_TIMER_BEGIN("grpc_mdelem_from_metadata_strings", 0);
+
+  gpr_mu_lock(&shard->mu);
+
+  idx = TABLE_IDX(hash, shard->capacity);
+  /* search for an existing pair */
+  for (md = shard->elems[idx]; md; md = md->bucket_next) {
+    if (grpc_slice_eq(key, md->key) && grpc_slice_eq(value, md->value)) {
+      REF_MD_LOCKED(shard, md);
+      gpr_mu_unlock(&shard->mu);
+      GPR_TIMER_END("grpc_mdelem_from_metadata_strings", 0);
+      return GRPC_MAKE_MDELEM(md, GRPC_MDELEM_STORAGE_INTERNED);
+    }
+  }
+
+  /* not found: create a new pair */
+  md = gpr_malloc(sizeof(interned_metadata));
+  gpr_atm_rel_store(&md->refcnt, 1);
+  md->key = grpc_slice_ref_internal(key);
+  md->value = grpc_slice_ref_internal(value);
+  md->user_data = 0;
+  md->destroy_user_data = 0;
+  md->bucket_next = shard->elems[idx];
+  shard->elems[idx] = md;
+  gpr_mu_init(&md->mu_user_data);
+#ifdef GRPC_METADATA_REFCOUNT_DEBUG
+  char *key_str = grpc_slice_to_c_string(md->key);
+  char *value_str = grpc_slice_to_c_string(md->value);
+  gpr_log(GPR_DEBUG, "ELM   NEW:%p:%zu: '%s' = '%s'", (void *)md,
+          gpr_atm_no_barrier_load(&md->refcnt), key_str, value_str);
+  gpr_free(key_str);
+  gpr_free(value_str);
+#endif
+  shard->count++;
+
+  if (shard->count > shard->capacity * 2) {
+    rehash_mdtab(exec_ctx, shard);
+  }
+
+  gpr_mu_unlock(&shard->mu);
+
+  GPR_TIMER_END("grpc_mdelem_from_metadata_strings", 0);
+
+  return GRPC_MAKE_MDELEM(md, GRPC_MDELEM_STORAGE_INTERNED);
+}
+
+grpc_mdelem grpc_mdelem_from_slices(grpc_exec_ctx *exec_ctx, grpc_slice key,
+                                    grpc_slice value) {
+  grpc_mdelem out = grpc_mdelem_create(exec_ctx, key, value, NULL);
+  grpc_slice_unref_internal(exec_ctx, key);
+  grpc_slice_unref_internal(exec_ctx, value);
+  return out;
+}
+
+grpc_mdelem grpc_mdelem_from_grpc_metadata(grpc_exec_ctx *exec_ctx,
+                                           grpc_metadata *metadata) {
+  bool changed = false;
+  grpc_slice key_slice =
+      grpc_slice_maybe_static_intern(metadata->key, &changed);
+  grpc_slice value_slice =
+      grpc_slice_maybe_static_intern(metadata->value, &changed);
+  return grpc_mdelem_create(exec_ctx, key_slice, value_slice,
+                            changed ? NULL : (grpc_mdelem_data *)metadata);
+}
+
+static size_t get_base64_encoded_size(size_t raw_length) {
+  static const uint8_t tail_xtra[3] = {0, 2, 3};
+  return raw_length / 3 * 4 + tail_xtra[raw_length % 3];
+}
+
+size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem elem) {
+  size_t overhead_and_key = 32 + GRPC_SLICE_LENGTH(GRPC_MDKEY(elem));
+  size_t value_len = GRPC_SLICE_LENGTH(GRPC_MDVALUE(elem));
+  if (grpc_is_binary_header(GRPC_MDKEY(elem))) {
+    return overhead_and_key + get_base64_encoded_size(value_len);
+  } else {
+    return overhead_and_key + value_len;
+  }
+}
+
+grpc_mdelem grpc_mdelem_ref(grpc_mdelem gmd DEBUG_ARGS) {
+  switch (GRPC_MDELEM_STORAGE(gmd)) {
+    case GRPC_MDELEM_STORAGE_EXTERNAL:
+    case GRPC_MDELEM_STORAGE_STATIC:
+      break;
+    case GRPC_MDELEM_STORAGE_INTERNED: {
+      interned_metadata *md = (interned_metadata *)GRPC_MDELEM_DATA(gmd);
+#ifdef GRPC_METADATA_REFCOUNT_DEBUG
+      char *key_str = grpc_slice_to_c_string(md->key);
+      char *value_str = grpc_slice_to_c_string(md->value);
+      gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+              "ELM   REF:%p:%zu->%zu: '%s' = '%s'", (void *)md,
+              gpr_atm_no_barrier_load(&md->refcnt),
+              gpr_atm_no_barrier_load(&md->refcnt) + 1, key_str, value_str);
+      gpr_free(key_str);
+      gpr_free(value_str);
+#endif
+      /* we can assume the ref count is >= 1 as the application is calling
+         this function - meaning that no adjustment to mdtab_free is necessary,
+         simplifying the logic here to be just an atomic increment */
+      /* use C assert to have this removed in opt builds */
+      GPR_ASSERT(gpr_atm_no_barrier_load(&md->refcnt) >= 1);
+      gpr_atm_no_barrier_fetch_add(&md->refcnt, 1);
+      break;
+    }
+    case GRPC_MDELEM_STORAGE_ALLOCATED: {
+      allocated_metadata *md = (allocated_metadata *)GRPC_MDELEM_DATA(gmd);
+#ifdef GRPC_METADATA_REFCOUNT_DEBUG
+      char *key_str = grpc_slice_to_c_string(md->key);
+      char *value_str = grpc_slice_to_c_string(md->value);
+      gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+              "ELM   REF:%p:%zu->%zu: '%s' = '%s'", (void *)md,
+              gpr_atm_no_barrier_load(&md->refcnt),
+              gpr_atm_no_barrier_load(&md->refcnt) + 1, key_str, value_str);
+      gpr_free(key_str);
+      gpr_free(value_str);
+#endif
+      /* we can assume the ref count is >= 1 as the application is calling
+         this function - meaning that no adjustment to mdtab_free is necessary,
+         simplifying the logic here to be just an atomic increment */
+      /* use C assert to have this removed in opt builds */
+      gpr_atm_no_barrier_fetch_add(&md->refcnt, 1);
+      break;
+    }
+  }
+  return gmd;
+}
+
+void grpc_mdelem_unref(grpc_exec_ctx *exec_ctx, grpc_mdelem gmd DEBUG_ARGS) {
+  switch (GRPC_MDELEM_STORAGE(gmd)) {
+    case GRPC_MDELEM_STORAGE_EXTERNAL:
+    case GRPC_MDELEM_STORAGE_STATIC:
+      break;
+    case GRPC_MDELEM_STORAGE_INTERNED: {
+      interned_metadata *md = (interned_metadata *)GRPC_MDELEM_DATA(gmd);
+#ifdef GRPC_METADATA_REFCOUNT_DEBUG
+      char *key_str = grpc_slice_to_c_string(md->key);
+      char *value_str = grpc_slice_to_c_string(md->value);
+      gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+              "ELM UNREF:%p:%zu->%zu: '%s' = '%s'", (void *)md,
+              gpr_atm_no_barrier_load(&md->refcnt),
+              gpr_atm_no_barrier_load(&md->refcnt) - 1, key_str, value_str);
+      gpr_free(key_str);
+      gpr_free(value_str);
+#endif
+      uint32_t hash = GRPC_MDSTR_KV_HASH(grpc_slice_hash(md->key),
+                                         grpc_slice_hash(md->value));
+      const gpr_atm prev_refcount = gpr_atm_full_fetch_add(&md->refcnt, -1);
+      GPR_ASSERT(prev_refcount >= 1);
+      if (1 == prev_refcount) {
+        /* once the refcount hits zero, some other thread can come along and
+           free md at any time: it's unsafe from this point on to access it */
+        mdtab_shard *shard = &g_shards[SHARD_IDX(hash)];
+        gpr_atm_no_barrier_fetch_add(&shard->free_estimate, 1);
+      }
+      break;
+    }
+    case GRPC_MDELEM_STORAGE_ALLOCATED: {
+      allocated_metadata *md = (allocated_metadata *)GRPC_MDELEM_DATA(gmd);
+#ifdef GRPC_METADATA_REFCOUNT_DEBUG
+      char *key_str = grpc_slice_to_c_string(md->key);
+      char *value_str = grpc_slice_to_c_string(md->value);
+      gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+              "ELM UNREF:%p:%zu->%zu: '%s' = '%s'", (void *)md,
+              gpr_atm_no_barrier_load(&md->refcnt),
+              gpr_atm_no_barrier_load(&md->refcnt) - 1, key_str, value_str);
+      gpr_free(key_str);
+      gpr_free(value_str);
+#endif
+      const gpr_atm prev_refcount = gpr_atm_full_fetch_add(&md->refcnt, -1);
+      GPR_ASSERT(prev_refcount >= 1);
+      if (1 == prev_refcount) {
+        grpc_slice_unref_internal(exec_ctx, md->key);
+        grpc_slice_unref_internal(exec_ctx, md->value);
+        gpr_free(md);
+      }
+      break;
+    }
+  }
+}
+
+void *grpc_mdelem_get_user_data(grpc_mdelem md, void (*destroy_func)(void *)) {
+  switch (GRPC_MDELEM_STORAGE(md)) {
+    case GRPC_MDELEM_STORAGE_EXTERNAL:
+    case GRPC_MDELEM_STORAGE_ALLOCATED:
+      return NULL;
+    case GRPC_MDELEM_STORAGE_STATIC:
+      return (void *)grpc_static_mdelem_user_data[GRPC_MDELEM_DATA(md) -
+                                                  grpc_static_mdelem_table];
+    case GRPC_MDELEM_STORAGE_INTERNED: {
+      interned_metadata *im = (interned_metadata *)GRPC_MDELEM_DATA(md);
+      void *result;
+      if (gpr_atm_acq_load(&im->destroy_user_data) == (gpr_atm)destroy_func) {
+        return (void *)gpr_atm_no_barrier_load(&im->user_data);
+      } else {
+        return NULL;
+      }
+      return result;
+    }
+  }
+  GPR_UNREACHABLE_CODE(return NULL);
+}
+
+void *grpc_mdelem_set_user_data(grpc_mdelem md, void (*destroy_func)(void *),
+                                void *user_data) {
+  switch (GRPC_MDELEM_STORAGE(md)) {
+    case GRPC_MDELEM_STORAGE_EXTERNAL:
+    case GRPC_MDELEM_STORAGE_ALLOCATED:
+      destroy_func(user_data);
+      return NULL;
+    case GRPC_MDELEM_STORAGE_STATIC:
+      destroy_func(user_data);
+      return (void *)grpc_static_mdelem_user_data[GRPC_MDELEM_DATA(md) -
+                                                  grpc_static_mdelem_table];
+    case GRPC_MDELEM_STORAGE_INTERNED: {
+      interned_metadata *im = (interned_metadata *)GRPC_MDELEM_DATA(md);
+      GPR_ASSERT(!is_mdelem_static(md));
+      GPR_ASSERT((user_data == NULL) == (destroy_func == NULL));
+      gpr_mu_lock(&im->mu_user_data);
+      if (gpr_atm_no_barrier_load(&im->destroy_user_data)) {
+        /* user data can only be set once */
+        gpr_mu_unlock(&im->mu_user_data);
+        if (destroy_func != NULL) {
+          destroy_func(user_data);
+        }
+        return (void *)gpr_atm_no_barrier_load(&im->user_data);
+      }
+      gpr_atm_no_barrier_store(&im->user_data, (gpr_atm)user_data);
+      gpr_atm_rel_store(&im->destroy_user_data, (gpr_atm)destroy_func);
+      gpr_mu_unlock(&im->mu_user_data);
+      return user_data;
+    }
+  }
+  GPR_UNREACHABLE_CODE(return NULL);
+}
+
+bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b) {
+  if (a.payload == b.payload) return true;
+  if (GRPC_MDELEM_IS_INTERNED(a) && GRPC_MDELEM_IS_INTERNED(b)) return false;
+  if (GRPC_MDISNULL(a) || GRPC_MDISNULL(b)) return false;
+  return grpc_slice_eq(GRPC_MDKEY(a), GRPC_MDKEY(b)) &&
+         grpc_slice_eq(GRPC_MDVALUE(a), GRPC_MDVALUE(b));
+}