grpc 1.15.0 → 1.30.0

data/src/core/lib/transport/metadata.h

@@ -21,22 +21,24 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <grpc/impl/codegen/log.h>
+
 #include <grpc/grpc.h>
 #include <grpc/slice.h>
 
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/gprpp/atomic.h"
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/slice/slice_utils.h"
 
 extern grpc_core::DebugOnlyTraceFlag grpc_trace_metadata;
 
 /* This file provides a mechanism for tracking metadata through the grpc stack.
    It's not intended for consumption outside of the library.
 
-   Metadata is tracked in the context of a grpc_mdctx. For the time being there
-   is one of these per-channel, avoiding cross channel interference with memory
-   use and lock contention.
-
-   The context tracks unique strings (grpc_mdstr) and pairs of strings
+   Metadata is tracked in the context of a sharded global grpc_mdctx. The
+   context tracks unique strings (grpc_mdstr) and pairs of strings
    (grpc_mdelem). Any of these objects can be checked for equality by comparing
    their pointers. These objects are reference counted.
 
@@ -63,7 +65,7 @@ extern grpc_core::DebugOnlyTraceFlag grpc_trace_metadata;
 typedef struct grpc_mdelem grpc_mdelem;
 
 /* if changing this, make identical changes in:
-   - interned_metadata, allocated_metadata in metadata.c
+   - grpc_core::{InternedMetadata, AllocatedMetadata}
    - grpc_metadata in grpc_types.h */
 typedef struct grpc_mdelem_data {
   const grpc_slice key;
@@ -75,17 +77,22 @@ typedef struct grpc_mdelem_data {
    this bit set in their integer value */
 #define GRPC_MDELEM_STORAGE_INTERNED_BIT 1
 
+/* External and static storage metadata has no refcount to ref/unref. Allocated
+ * and interned metadata do have a refcount. Metadata ref and unref methods use
+ * a switch statement on this enum to determine which behaviour to execute.
+ * Keeping the no-ref cases together and the ref-cases together leads to
+ * slightly better code generation (9 inlined instructions rather than 10). */
 typedef enum {
   /* memory pointed to by grpc_mdelem::payload is owned by an external system */
   GRPC_MDELEM_STORAGE_EXTERNAL = 0,
-  /* memory pointed to by grpc_mdelem::payload is interned by the metadata
-     system */
-  GRPC_MDELEM_STORAGE_INTERNED = GRPC_MDELEM_STORAGE_INTERNED_BIT,
+  /* memory is in the static metadata table */
+  GRPC_MDELEM_STORAGE_STATIC = GRPC_MDELEM_STORAGE_INTERNED_BIT,
   /* memory pointed to by grpc_mdelem::payload is allocated by the metadata
      system */
   GRPC_MDELEM_STORAGE_ALLOCATED = 2,
-  /* memory is in the static metadata table */
-  GRPC_MDELEM_STORAGE_STATIC = 2 | GRPC_MDELEM_STORAGE_INTERNED_BIT,
+  /* memory pointed to by grpc_mdelem::payload is interned by the metadata
+     system */
+  GRPC_MDELEM_STORAGE_INTERNED = 2 | GRPC_MDELEM_STORAGE_INTERNED_BIT,
 } grpc_mdelem_data_storage;
 
 struct grpc_mdelem {
@@ -108,8 +115,30 @@ struct grpc_mdelem {
   ((grpc_mdelem_data_storage)((md).payload & \
                               (uintptr_t)GRPC_MDELEM_STORAGE_INTERNED_BIT))
 
-/* Unrefs the slices. */
-grpc_mdelem grpc_mdelem_from_slices(grpc_slice key, grpc_slice value);
+/* Given arbitrary input slices, create a grpc_mdelem object. The caller refs
+ * the input slices; we unref them. This method is always safe to call; however,
+ * if we know data about the slices in question (e.g. if we knew our key was
+ * static) we can call specializations that save on cycle count. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_slice& key,
+                                    const grpc_slice& value);
+
+/* Like grpc_mdelem_from_slices, but we know that key is a static slice. This
+   saves us a few branches and a no-op call to md_unref() for the key. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_core::StaticMetadataSlice& key,
+                                    const grpc_slice& value);
+
+/* Like grpc_mdelem_from_slices, but key is static and val is static. */
+grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::StaticMetadataSlice& key,
+    const grpc_core::StaticMetadataSlice& value);
+
+/* Like grpc_mdelem_from_slices, but key is static and val is interned. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_core::StaticMetadataSlice& key,
+                                    const grpc_core::ManagedMemorySlice& value);
+
+/* Like grpc_mdelem_from_slices, but key and val are interned. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_core::ManagedMemorySlice& key,
+                                    const grpc_core::ManagedMemorySlice& value);
 
 /* Cheaply convert a grpc_metadata to a grpc_mdelem; may use the grpc_metadata
    object as backing storage (so lifetimes should align) */
@@ -120,37 +149,258 @@ grpc_mdelem grpc_mdelem_from_grpc_metadata(grpc_metadata* metadata);
    compatible_external_backing_store if it is non-NULL (in which case it's the
    users responsibility to ensure that it outlives usage) */
 grpc_mdelem grpc_mdelem_create(
-    grpc_slice key, grpc_slice value,
+    const grpc_slice& key, const grpc_slice& value,
+    grpc_mdelem_data* compatible_external_backing_store);
+
+/* Like grpc_mdelem_create, but we know that key is static. */
+grpc_mdelem grpc_mdelem_create(
+    const grpc_core::StaticMetadataSlice& key, const grpc_slice& value,
     grpc_mdelem_data* compatible_external_backing_store);
 
+#define GRPC_MDKEY(md) (GRPC_MDELEM_DATA(md)->key)
+#define GRPC_MDVALUE(md) (GRPC_MDELEM_DATA(md)->value)
+
 bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b);
+/* Often we compare metadata where we know a-priori that the second parameter is
+ * static, and that the keys match. This most commonly happens when processing
+ * metadata batch callouts in initial/trailing filters. In this case, fastpath
+ * grpc_mdelem_eq and remove unnecessary checks. */
+inline bool grpc_mdelem_static_value_eq(grpc_mdelem a, grpc_mdelem b_static) {
+  if (a.payload == b_static.payload) return true;
+  return grpc_slice_eq_static_interned(GRPC_MDVALUE(a), GRPC_MDVALUE(b_static));
+}
+#define GRPC_MDISNULL(md) (GRPC_MDELEM_DATA(md) == NULL)
 
-size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem elem,
-                                           bool use_true_binary_metadata);
+inline bool grpc_mdelem_both_interned_eq(grpc_mdelem a_interned,
+                                         grpc_mdelem b_interned) {
+  GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(a_interned) ||
+                   GRPC_MDISNULL(a_interned));
+  GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(b_interned) ||
+                   GRPC_MDISNULL(b_interned));
+  return a_interned.payload == b_interned.payload;
+}
 
 /* Mutator and accessor for grpc_mdelem user data. The destructor function
    is used as a type tag and is checked during user_data fetch. */
 void* grpc_mdelem_get_user_data(grpc_mdelem md, void (*if_destroy_func)(void*));
 void* grpc_mdelem_set_user_data(grpc_mdelem md, void (*destroy_func)(void*),
-                                void* user_data);
+                                void* data);
+
+// Defined in metadata.cc.
+struct mdtab_shard;
+
+#ifndef NDEBUG
+void grpc_mdelem_trace_ref(void* md, const grpc_slice& key,
+                           const grpc_slice& value, intptr_t refcnt,
+                           const char* file, int line);
+void grpc_mdelem_trace_unref(void* md, const grpc_slice& key,
+                             const grpc_slice& value, intptr_t refcnt,
+                             const char* file, int line);
+#endif
+namespace grpc_core {
+
+typedef void (*destroy_user_data_func)(void* data);
+
+struct UserData {
+  Mutex mu_user_data;
+  grpc_core::Atomic<destroy_user_data_func> destroy_user_data;
+  grpc_core::Atomic<void*> data;
+};
+
+class StaticMetadata {
+ public:
+  StaticMetadata(const grpc_slice& key, const grpc_slice& value, uintptr_t idx)
+      : kv_({key, value}), hash_(0), static_idx_(idx) {}
+
+  const grpc_mdelem_data& data() const { return kv_; }
+
+  void HashInit();
+  uint32_t hash() { return hash_; }
+  uintptr_t StaticIndex() { return static_idx_; }
+
+ private:
+  grpc_mdelem_data kv_;
+
+  /* private only data */
+  uint32_t hash_;
+  uintptr_t static_idx_;
+};
+
+class RefcountedMdBase {
+ public:
+  RefcountedMdBase(const grpc_slice& key, const grpc_slice& value)
+      : key_(key), value_(value), refcnt_(1) {}
+  RefcountedMdBase(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash)
+      : key_(key), value_(value), refcnt_(1), hash_(hash) {}
+
+  const grpc_slice& key() const { return key_; }
+  const grpc_slice& value() const { return value_; }
+  uint32_t hash() { return hash_; }
+
+#ifndef NDEBUG
+  void Ref(const char* file, int line) {
+    grpc_mdelem_trace_ref(this, key_, value_, RefValue(), file, line);
+    const intptr_t prior = refcnt_.FetchAdd(1, MemoryOrder::RELAXED);
+    GPR_ASSERT(prior > 0);
+  }
+  bool Unref(const char* file, int line) {
+    grpc_mdelem_trace_unref(this, key_, value_, RefValue(), file, line);
+    return Unref();
+  }
+#endif
+  void Ref() {
+    /* we can assume the ref count is >= 1 as the application is calling
+       this function - meaning that no adjustment to mdtab_free is necessary,
+       simplifying the logic here to be just an atomic increment */
+    refcnt_.FetchAdd(1, MemoryOrder::RELAXED);
+  }
+  bool Unref() {
+    const intptr_t prior = refcnt_.FetchSub(1, MemoryOrder::ACQ_REL);
+    GPR_DEBUG_ASSERT(prior > 0);
+    return prior == 1;
+  }
+
+ protected:
+#ifndef NDEBUG
+  void TraceAtStart(const char* tag);
+#endif
+
+  intptr_t RefValue() { return refcnt_.Load(MemoryOrder::RELAXED); }
+  bool AllRefsDropped() { return refcnt_.Load(MemoryOrder::ACQUIRE) == 0; }
+  bool FirstRef() { return refcnt_.FetchAdd(1, MemoryOrder::RELAXED) == 0; }
+
+ private:
+  /* must be byte compatible with grpc_mdelem_data */
+  grpc_slice key_;
+  grpc_slice value_;
+  grpc_core::Atomic<intptr_t> refcnt_;
+  uint32_t hash_ = 0;
+};
+
+class InternedMetadata : public RefcountedMdBase {
+ public:
+  // TODO(arjunroy): Change to use strongly typed slices instead.
+  struct NoRefKey {};
+  struct BucketLink {
+    explicit BucketLink(InternedMetadata* md) : next(md) {}
+
+    InternedMetadata* next = nullptr;
+  };
+  InternedMetadata(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash, InternedMetadata* next);
+  InternedMetadata(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash, InternedMetadata* next, const NoRefKey*);
+
+  ~InternedMetadata();
+  void RefWithShardLocked(mdtab_shard* shard);
+  UserData* user_data() { return &user_data_; }
+  InternedMetadata* bucket_next() { return link_.next; }
+  void set_bucket_next(InternedMetadata* md) { link_.next = md; }
+
+  static size_t CleanupLinkedMetadata(BucketLink* head);
+
+ private:
+  UserData user_data_;
+  BucketLink link_;
+};
+
+/* Shadow structure for grpc_mdelem_data for allocated elements */
+class AllocatedMetadata : public RefcountedMdBase {
+ public:
+  // TODO(arjunroy): Change to use strongly typed slices instead.
+  struct NoRefKey {};
+  AllocatedMetadata(const grpc_slice& key, const grpc_slice& value);
+  AllocatedMetadata(const grpc_core::ManagedMemorySlice& key,
+                    const grpc_core::UnmanagedMemorySlice& value);
+  AllocatedMetadata(const grpc_core::ExternallyManagedSlice& key,
+                    const grpc_core::UnmanagedMemorySlice& value);
+  AllocatedMetadata(const grpc_slice& key, const grpc_slice& value,
+                    const NoRefKey*);
+  ~AllocatedMetadata();
+
+  UserData* user_data() { return &user_data_; }
+
+ private:
+  UserData user_data_;
+};
+
+}  // namespace grpc_core
 
 #ifndef NDEBUG
 #define GRPC_MDELEM_REF(s) grpc_mdelem_ref((s), __FILE__, __LINE__)
+inline grpc_mdelem grpc_mdelem_ref(grpc_mdelem gmd, const char* file,
+                                   int line) {
+#else  // ifndef NDEBUG
+#define GRPC_MDELEM_REF(s) grpc_mdelem_ref((s))
+inline grpc_mdelem grpc_mdelem_ref(grpc_mdelem gmd) {
+#endif  // ifndef NDEBUG
+  switch (GRPC_MDELEM_STORAGE(gmd)) {
+    case GRPC_MDELEM_STORAGE_EXTERNAL:
+    case GRPC_MDELEM_STORAGE_STATIC:
+      break;
+    case GRPC_MDELEM_STORAGE_INTERNED: {
+      auto* md =
+          reinterpret_cast<grpc_core::InternedMetadata*> GRPC_MDELEM_DATA(gmd);
+      /* use C assert to have this removed in opt builds */
+#ifndef NDEBUG
+      md->Ref(file, line);
+#else
+      md->Ref();
+#endif
+      break;
+    }
+    case GRPC_MDELEM_STORAGE_ALLOCATED: {
+      auto* md =
+          reinterpret_cast<grpc_core::AllocatedMetadata*> GRPC_MDELEM_DATA(gmd);
+#ifndef NDEBUG
+      md->Ref(file, line);
+#else
+      md->Ref();
+#endif
+      break;
+    }
+  }
+  return gmd;
+}
+
+#ifndef NDEBUG
 #define GRPC_MDELEM_UNREF(s) grpc_mdelem_unref((s), __FILE__, __LINE__)
-grpc_mdelem grpc_mdelem_ref(grpc_mdelem md, const char* file, int line);
-void grpc_mdelem_unref(grpc_mdelem md, const char* file, int line);
+void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
+                                uint32_t hash, const char* file, int line);
+inline void grpc_mdelem_unref(grpc_mdelem gmd, const char* file, int line) {
 #else
-#define GRPC_MDELEM_REF(s) grpc_mdelem_ref((s))
 #define GRPC_MDELEM_UNREF(s) grpc_mdelem_unref((s))
-grpc_mdelem grpc_mdelem_ref(grpc_mdelem md);
-void grpc_mdelem_unref(grpc_mdelem md);
+void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
+                                uint32_t hash);
+inline void grpc_mdelem_unref(grpc_mdelem gmd) {
 #endif
-
-#define GRPC_MDKEY(md) (GRPC_MDELEM_DATA(md)->key)
-#define GRPC_MDVALUE(md) (GRPC_MDELEM_DATA(md)->value)
+  const grpc_mdelem_data_storage storage = GRPC_MDELEM_STORAGE(gmd);
+  switch (storage) {
+    case GRPC_MDELEM_STORAGE_EXTERNAL:
+    case GRPC_MDELEM_STORAGE_STATIC:
+      return;
+    case GRPC_MDELEM_STORAGE_INTERNED:
+    case GRPC_MDELEM_STORAGE_ALLOCATED:
+      auto* md =
+          reinterpret_cast<grpc_core::RefcountedMdBase*> GRPC_MDELEM_DATA(gmd);
+      /* once the refcount hits zero, some other thread can come along and
+         free an interned md at any time: it's unsafe from this point on to
+         access it so we read the hash now. */
+      uint32_t hash = md->hash();
+#ifndef NDEBUG
+      if (GPR_UNLIKELY(md->Unref(file, line))) {
+        grpc_mdelem_on_final_unref(storage, md, hash, file, line);
+#else
+      if (GPR_UNLIKELY(md->Unref())) {
+        grpc_mdelem_on_final_unref(storage, md, hash);
+#endif
+      }
+      return;
+  }
+}
 
 #define GRPC_MDNULL GRPC_MAKE_MDELEM(NULL, GRPC_MDELEM_STORAGE_EXTERNAL)
-#define GRPC_MDISNULL(md) (GRPC_MDELEM_DATA(md) == NULL)
 
 /* We add 32 bytes of padding as per RFC-7540 section 6.5.2. */
 #define GRPC_MDELEM_LENGTH(e)                                                  \
@@ -162,4 +412,35 @@ void grpc_mdelem_unref(grpc_mdelem md);
 void grpc_mdctx_global_init(void);
 void grpc_mdctx_global_shutdown();
 
+/* Like grpc_mdelem_from_slices, but we know that key is a static or interned
+   slice and value is not static or interned. This gives us an inlinable
+   fastpath - we know we must allocate metadata now, and that we do not need to
+   unref the value (rather, we just transfer the ref). We can avoid a ref since:
+   1) the key slice is passed in already ref'd
+   2) We're guaranteed to create a new Allocated slice, thus meaning the
+      ref can be considered 'transferred'.*/
+inline grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::ManagedMemorySlice& key,
+    const grpc_core::UnmanagedMemorySlice& value) {
+  using grpc_core::AllocatedMetadata;
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
+                          GRPC_MDELEM_STORAGE_ALLOCATED);
+}
+
+inline grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::ExternallyManagedSlice& key,
+    const grpc_core::UnmanagedMemorySlice& value) {
+  using grpc_core::AllocatedMetadata;
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
+                          GRPC_MDELEM_STORAGE_ALLOCATED);
+}
+
+inline grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::StaticMetadataSlice& key,
+    const grpc_core::UnmanagedMemorySlice& value) {
+  using grpc_core::AllocatedMetadata;
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
+                          GRPC_MDELEM_STORAGE_ALLOCATED);
+}
+
 #endif /* GRPC_CORE_LIB_TRANSPORT_METADATA_H */

data/src/core/lib/transport/metadata_batch.cc

@@ -50,6 +50,9 @@ static void assert_valid_list(grpc_mdelem_list* list) {
     verified_count++;
   }
   GPR_ASSERT(list->count == verified_count);
+#else
+  // Avoid unused-parameter warning for debug-only parameter
+  (void)list;
 #endif /* NDEBUG */
 }
 
@@ -64,6 +67,9 @@ static void assert_valid_callouts(grpc_metadata_batch* batch) {
     }
     grpc_slice_unref_internal(key_interned);
   }
+#else
+  // Avoid unused-parameter warning for debug-only parameter
+  (void)batch;
 #endif
 }
 
@@ -93,6 +99,23 @@ grpc_error* grpc_attach_md_to_error(grpc_error* src, grpc_mdelem md) {
   return out;
 }
 
+static grpc_error* GPR_ATTRIBUTE_NOINLINE error_with_md(grpc_mdelem md) {
+  return grpc_attach_md_to_error(
+      GRPC_ERROR_CREATE_FROM_STATIC_STRING("Unallowed duplicate metadata"), md);
+}
+
+static grpc_error* link_callout(grpc_metadata_batch* batch,
+                                grpc_linked_mdelem* storage,
+                                grpc_metadata_batch_callouts_index idx) {
+  GPR_DEBUG_ASSERT(idx >= 0 && idx < GRPC_BATCH_CALLOUTS_COUNT);
+  if (GPR_LIKELY(batch->idx.array[idx] == nullptr)) {
+    ++batch->list.default_count;
+    batch->idx.array[idx] = storage;
+    return GRPC_ERROR_NONE;
+  }
+  return error_with_md(storage->md);
+}
+
 static grpc_error* maybe_link_callout(grpc_metadata_batch* batch,
                                       grpc_linked_mdelem* storage)
     GRPC_MUST_USE_RESULT;
@@ -104,14 +127,7 @@ static grpc_error* maybe_link_callout(grpc_metadata_batch* batch,
   if (idx == GRPC_BATCH_CALLOUTS_COUNT) {
     return GRPC_ERROR_NONE;
   }
-  if (batch->idx.array[idx] == nullptr) {
-    if (grpc_static_callout_is_default[idx]) ++batch->list.default_count;
-    batch->idx.array[idx] = storage;
-    return GRPC_ERROR_NONE;
-  }
-  return grpc_attach_md_to_error(
-      GRPC_ERROR_CREATE_FROM_STATIC_STRING("Unallowed duplicate metadata"),
-      storage->md);
+  return link_callout(batch, storage, idx);
 }
 
 static void maybe_unlink_callout(grpc_metadata_batch* batch,
@@ -121,24 +137,25 @@ static void maybe_unlink_callout(grpc_metadata_batch* batch,
   if (idx == GRPC_BATCH_CALLOUTS_COUNT) {
     return;
   }
-  if (grpc_static_callout_is_default[idx]) --batch->list.default_count;
-  GPR_ASSERT(batch->idx.array[idx] != nullptr);
+  --batch->list.default_count;
+  GPR_DEBUG_ASSERT(batch->idx.array[idx] != nullptr);
   batch->idx.array[idx] = nullptr;
 }
 
 grpc_error* grpc_metadata_batch_add_head(grpc_metadata_batch* batch,
                                          grpc_linked_mdelem* storage,
                                          grpc_mdelem elem_to_add) {
-  GPR_ASSERT(!GRPC_MDISNULL(elem_to_add));
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(elem_to_add));
   storage->md = elem_to_add;
   return grpc_metadata_batch_link_head(batch, storage);
 }
 
 static void link_head(grpc_mdelem_list* list, grpc_linked_mdelem* storage) {
   assert_valid_list(list);
-  GPR_ASSERT(!GRPC_MDISNULL(storage->md));
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(storage->md));
   storage->prev = nullptr;
   storage->next = list->head;
+  storage->reserved = nullptr;
   if (list->head != nullptr) {
     list->head->prev = storage;
   } else {
@@ -162,17 +179,35 @@ grpc_error* grpc_metadata_batch_link_head(grpc_metadata_batch* batch,
   return GRPC_ERROR_NONE;
 }
 
+// TODO(arjunroy): Need to revisit this and see what guarantees exist between
+// C-core and the internal-metadata subsystem. E.g. can we ensure a particular
+// metadata is never added twice, even in the presence of user supplied data?
+grpc_error* grpc_metadata_batch_link_head(
+    grpc_metadata_batch* batch, grpc_linked_mdelem* storage,
+    grpc_metadata_batch_callouts_index idx) {
+  GPR_DEBUG_ASSERT(GRPC_BATCH_INDEX_OF(GRPC_MDKEY(storage->md)) == idx);
+  assert_valid_callouts(batch);
+  grpc_error* err = link_callout(batch, storage, idx);
+  if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) {
+    assert_valid_callouts(batch);
+    return err;
+  }
+  link_head(&batch->list, storage);
+  assert_valid_callouts(batch);
+  return GRPC_ERROR_NONE;
+}
+
 grpc_error* grpc_metadata_batch_add_tail(grpc_metadata_batch* batch,
                                          grpc_linked_mdelem* storage,
                                          grpc_mdelem elem_to_add) {
-  GPR_ASSERT(!GRPC_MDISNULL(elem_to_add));
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(elem_to_add));
   storage->md = elem_to_add;
   return grpc_metadata_batch_link_tail(batch, storage);
 }
 
 static void link_tail(grpc_mdelem_list* list, grpc_linked_mdelem* storage) {
   assert_valid_list(list);
-  GPR_ASSERT(!GRPC_MDISNULL(storage->md));
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(storage->md));
   storage->prev = list->tail;
   storage->next = nullptr;
   storage->reserved = nullptr;
@@ -199,6 +234,21 @@ grpc_error* grpc_metadata_batch_link_tail(grpc_metadata_batch* batch,
   return GRPC_ERROR_NONE;
 }
 
+grpc_error* grpc_metadata_batch_link_tail(
+    grpc_metadata_batch* batch, grpc_linked_mdelem* storage,
+    grpc_metadata_batch_callouts_index idx) {
+  GPR_DEBUG_ASSERT(GRPC_BATCH_INDEX_OF(GRPC_MDKEY(storage->md)) == idx);
+  assert_valid_callouts(batch);
+  grpc_error* err = link_callout(batch, storage, idx);
+  if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) {
+    assert_valid_callouts(batch);
+    return err;
+  }
+  link_tail(&batch->list, storage);
+  assert_valid_callouts(batch);
+  return GRPC_ERROR_NONE;
+}
+
 static void unlink_storage(grpc_mdelem_list* list,
                            grpc_linked_mdelem* storage) {
   assert_valid_list(list);
@@ -225,8 +275,20 @@ void grpc_metadata_batch_remove(grpc_metadata_batch* batch,
   assert_valid_callouts(batch);
 }
 
+void grpc_metadata_batch_remove(grpc_metadata_batch* batch,
+                                grpc_metadata_batch_callouts_index idx) {
+  assert_valid_callouts(batch);
+  grpc_linked_mdelem* storage = batch->idx.array[idx];
+  GPR_DEBUG_ASSERT(storage != nullptr);
+  --batch->list.default_count;
+  batch->idx.array[idx] = nullptr;
+  unlink_storage(&batch->list, storage);
+  GRPC_MDELEM_UNREF(storage->md);
+  assert_valid_callouts(batch);
+}
+
 void grpc_metadata_batch_set_value(grpc_linked_mdelem* storage,
-                                   grpc_slice value) {
+                                   const grpc_slice& value) {
   grpc_mdelem old_mdelem = storage->md;
   grpc_mdelem new_mdelem = grpc_mdelem_from_slices(
       grpc_slice_ref_internal(GRPC_MDKEY(old_mdelem)), value);
@@ -312,13 +374,14 @@ void grpc_metadata_batch_copy(grpc_metadata_batch* src,
   size_t i = 0;
   for (grpc_linked_mdelem* elem = src->list.head; elem != nullptr;
        elem = elem->next) {
-    grpc_error* error = grpc_metadata_batch_add_tail(dst, &storage[i++],
-                                                     GRPC_MDELEM_REF(elem->md));
+    // Error unused in non-debug builds.
+    grpc_error* GRPC_UNUSED error = grpc_metadata_batch_add_tail(
+        dst, &storage[i++], GRPC_MDELEM_REF(elem->md));
     // The only way that grpc_metadata_batch_add_tail() can fail is if
     // there's a duplicate entry for a callout.  However, that can't be
     // the case here, because we would not have been allowed to create
     // a source batch that had that kind of conflict.
-    GPR_ASSERT(error == GRPC_ERROR_NONE);
+    GPR_DEBUG_ASSERT(error == GRPC_ERROR_NONE);
   }
 }