grpc 1.21.0 → 1.32.0

data/src/core/lib/transport/metadata.h

@@ -21,7 +21,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "include/grpc/impl/codegen/log.h"
+#include <grpc/impl/codegen/log.h>
 
 #include <grpc/grpc.h>
 #include <grpc/slice.h>
@@ -30,17 +30,15 @@
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/atomic.h"
 #include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/slice/slice_utils.h"
 
 extern grpc_core::DebugOnlyTraceFlag grpc_trace_metadata;
 
 /* This file provides a mechanism for tracking metadata through the grpc stack.
    It's not intended for consumption outside of the library.
 
-   Metadata is tracked in the context of a grpc_mdctx. For the time being there
-   is one of these per-channel, avoiding cross channel interference with memory
-   use and lock contention.
-
-   The context tracks unique strings (grpc_mdstr) and pairs of strings
+   Metadata is tracked in the context of a sharded global grpc_mdctx. The
+   context tracks unique strings (grpc_mdstr) and pairs of strings
    (grpc_mdelem). Any of these objects can be checked for equality by comparing
    their pointers. These objects are reference counted.
 
@@ -79,17 +77,22 @@ typedef struct grpc_mdelem_data {
    this bit set in their integer value */
 #define GRPC_MDELEM_STORAGE_INTERNED_BIT 1
 
+/* External and static storage metadata has no refcount to ref/unref. Allocated
+ * and interned metadata do have a refcount. Metadata ref and unref methods use
+ * a switch statement on this enum to determine which behaviour to execute.
+ * Keeping the no-ref cases together and the ref-cases together leads to
+ * slightly better code generation (9 inlined instructions rather than 10). */
 typedef enum {
   /* memory pointed to by grpc_mdelem::payload is owned by an external system */
   GRPC_MDELEM_STORAGE_EXTERNAL = 0,
-  /* memory pointed to by grpc_mdelem::payload is interned by the metadata
-     system */
-  GRPC_MDELEM_STORAGE_INTERNED = GRPC_MDELEM_STORAGE_INTERNED_BIT,
+  /* memory is in the static metadata table */
+  GRPC_MDELEM_STORAGE_STATIC = GRPC_MDELEM_STORAGE_INTERNED_BIT,
   /* memory pointed to by grpc_mdelem::payload is allocated by the metadata
      system */
   GRPC_MDELEM_STORAGE_ALLOCATED = 2,
-  /* memory is in the static metadata table */
-  GRPC_MDELEM_STORAGE_STATIC = 2 | GRPC_MDELEM_STORAGE_INTERNED_BIT,
+  /* memory pointed to by grpc_mdelem::payload is interned by the metadata
+     system */
+  GRPC_MDELEM_STORAGE_INTERNED = 2 | GRPC_MDELEM_STORAGE_INTERNED_BIT,
 } grpc_mdelem_data_storage;
 
 struct grpc_mdelem {
@@ -112,10 +115,31 @@ struct grpc_mdelem {
   ((grpc_mdelem_data_storage)((md).payload & \
                               (uintptr_t)GRPC_MDELEM_STORAGE_INTERNED_BIT))
 
-/* Unrefs the slices. */
+/* Given arbitrary input slices, create a grpc_mdelem object. The caller refs
+ * the input slices; we unref them. This method is always safe to call; however,
+ * if we know data about the slices in question (e.g. if we knew our key was
+ * static) we can call specializations that save on cycle count. */
 grpc_mdelem grpc_mdelem_from_slices(const grpc_slice& key,
                                     const grpc_slice& value);
 
+/* Like grpc_mdelem_from_slices, but we know that key is a static slice. This
+   saves us a few branches and a no-op call to md_unref() for the key. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_core::StaticMetadataSlice& key,
+                                    const grpc_slice& value);
+
+/* Like grpc_mdelem_from_slices, but key is static and val is static. */
+grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::StaticMetadataSlice& key,
+    const grpc_core::StaticMetadataSlice& value);
+
+/* Like grpc_mdelem_from_slices, but key is static and val is interned. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_core::StaticMetadataSlice& key,
+                                    const grpc_core::ManagedMemorySlice& value);
+
+/* Like grpc_mdelem_from_slices, but key and val are interned. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_core::ManagedMemorySlice& key,
+                                    const grpc_core::ManagedMemorySlice& value);
+
 /* Cheaply convert a grpc_metadata to a grpc_mdelem; may use the grpc_metadata
    object as backing storage (so lifetimes should align) */
 grpc_mdelem grpc_mdelem_from_grpc_metadata(grpc_metadata* metadata);
@@ -128,6 +152,11 @@ grpc_mdelem grpc_mdelem_create(
     const grpc_slice& key, const grpc_slice& value,
     grpc_mdelem_data* compatible_external_backing_store);
 
+/* Like grpc_mdelem_create, but we know that key is static. */
+grpc_mdelem grpc_mdelem_create(
+    const grpc_core::StaticMetadataSlice& key, const grpc_slice& value,
+    grpc_mdelem_data* compatible_external_backing_store);
+
 #define GRPC_MDKEY(md) (GRPC_MDELEM_DATA(md)->key)
 #define GRPC_MDVALUE(md) (GRPC_MDELEM_DATA(md)->value)
 
@@ -138,7 +167,17 @@ bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b);
  * grpc_mdelem_eq and remove unnecessary checks. */
 inline bool grpc_mdelem_static_value_eq(grpc_mdelem a, grpc_mdelem b_static) {
   if (a.payload == b_static.payload) return true;
-  return grpc_slice_eq(GRPC_MDVALUE(a), GRPC_MDVALUE(b_static));
+  return grpc_slice_eq_static_interned(GRPC_MDVALUE(a), GRPC_MDVALUE(b_static));
+}
+#define GRPC_MDISNULL(md) (GRPC_MDELEM_DATA(md) == NULL)
+
+inline bool grpc_mdelem_both_interned_eq(grpc_mdelem a_interned,
+                                         grpc_mdelem b_interned) {
+  GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(a_interned) ||
+                   GRPC_MDISNULL(a_interned));
+  GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(b_interned) ||
+                   GRPC_MDISNULL(b_interned));
+  return a_interned.payload == b_interned.payload;
 }
 
 /* Mutator and accessor for grpc_mdelem user data. The destructor function
@@ -168,17 +207,36 @@ struct UserData {
   grpc_core::Atomic<void*> data;
 };
 
-class InternedMetadata {
+class StaticMetadata {
  public:
-  struct BucketLink {
-    explicit BucketLink(InternedMetadata* md) : next(md) {}
+  StaticMetadata(const grpc_slice& key, const grpc_slice& value, uintptr_t idx)
+      : kv_({key, value}), hash_(0), static_idx_(idx) {}
 
-    InternedMetadata* next = nullptr;
-  };
+  const grpc_mdelem_data& data() const { return kv_; }
 
-  InternedMetadata(const grpc_slice& key, const grpc_slice& value,
-                   uint32_t hash, InternedMetadata* next);
-  ~InternedMetadata();
+  void HashInit();
+  uint32_t hash() { return hash_; }
+  uintptr_t StaticIndex() { return static_idx_; }
+
+ private:
+  grpc_mdelem_data kv_;
+
+  /* private only data */
+  uint32_t hash_;
+  uintptr_t static_idx_;
+};
+
+class RefcountedMdBase {
+ public:
+  RefcountedMdBase(const grpc_slice& key, const grpc_slice& value)
+      : key_(key), value_(value), refcnt_(1) {}
+  RefcountedMdBase(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash)
+      : key_(key), value_(value), refcnt_(1), hash_(hash) {}
+
+  const grpc_slice& key() const { return key_; }
+  const grpc_slice& value() const { return value_; }
+  uint32_t hash() { return hash_; }
 
 #ifndef NDEBUG
   void Ref(const char* file, int line) {
@@ -190,92 +248,80 @@ class InternedMetadata {
     grpc_mdelem_trace_unref(this, key_, value_, RefValue(), file, line);
     return Unref();
   }
-#else
-  // We define a naked Ref() in the else-clause to make sure we don't
-  // inadvertently skip the assert on debug builds.
+#endif
   void Ref() {
     /* we can assume the ref count is >= 1 as the application is calling
        this function - meaning that no adjustment to mdtab_free is necessary,
        simplifying the logic here to be just an atomic increment */
     refcnt_.FetchAdd(1, MemoryOrder::RELAXED);
   }
-#endif  // ifndef NDEBUG
   bool Unref() {
     const intptr_t prior = refcnt_.FetchSub(1, MemoryOrder::ACQ_REL);
     GPR_DEBUG_ASSERT(prior > 0);
     return prior == 1;
   }
 
-  void RefWithShardLocked(mdtab_shard* shard);
-  const grpc_slice& key() const { return key_; }
-  const grpc_slice& value() const { return value_; }
-  UserData* user_data() { return &user_data_; }
-  uint32_t hash() { return hash_; }
-  InternedMetadata* bucket_next() { return link_.next; }
-  void set_bucket_next(InternedMetadata* md) { link_.next = md; }
-
-  static size_t CleanupLinkedMetadata(BucketLink* head);
+ protected:
+#ifndef NDEBUG
+  void TraceAtStart(const char* tag);
+#endif
 
- private:
+  intptr_t RefValue() { return refcnt_.Load(MemoryOrder::RELAXED); }
   bool AllRefsDropped() { return refcnt_.Load(MemoryOrder::ACQUIRE) == 0; }
   bool FirstRef() { return refcnt_.FetchAdd(1, MemoryOrder::RELAXED) == 0; }
-  intptr_t RefValue() { return refcnt_.Load(MemoryOrder::RELAXED); }
 
+ private:
   /* must be byte compatible with grpc_mdelem_data */
   grpc_slice key_;
   grpc_slice value_;
-
-  /* private only data */
   grpc_core::Atomic<intptr_t> refcnt_;
-  uint32_t hash_;
+  uint32_t hash_ = 0;
+};
 
-  UserData user_data_;
+class InternedMetadata : public RefcountedMdBase {
+ public:
+  // TODO(arjunroy): Change to use strongly typed slices instead.
+  struct NoRefKey {};
+  struct BucketLink {
+    explicit BucketLink(InternedMetadata* md) : next(md) {}
+
+    InternedMetadata* next = nullptr;
+  };
+  InternedMetadata(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash, InternedMetadata* next);
+  InternedMetadata(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash, InternedMetadata* next, const NoRefKey*);
+
+  ~InternedMetadata();
+  void RefWithShardLocked(mdtab_shard* shard);
+  UserData* user_data() { return &user_data_; }
+  InternedMetadata* bucket_next() { return link_.next; }
+  void set_bucket_next(InternedMetadata* md) { link_.next = md; }
 
+  static size_t CleanupLinkedMetadata(BucketLink* head);
+
+ private:
+  UserData user_data_;
   BucketLink link_;
 };
 
 /* Shadow structure for grpc_mdelem_data for allocated elements */
-class AllocatedMetadata {
+class AllocatedMetadata : public RefcountedMdBase {
  public:
+  // TODO(arjunroy): Change to use strongly typed slices instead.
+  struct NoRefKey {};
   AllocatedMetadata(const grpc_slice& key, const grpc_slice& value);
+  AllocatedMetadata(const grpc_core::ManagedMemorySlice& key,
+                    const grpc_core::UnmanagedMemorySlice& value);
+  AllocatedMetadata(const grpc_core::ExternallyManagedSlice& key,
+                    const grpc_core::UnmanagedMemorySlice& value);
+  AllocatedMetadata(const grpc_slice& key, const grpc_slice& value,
+                    const NoRefKey*);
   ~AllocatedMetadata();
 
-  const grpc_slice& key() const { return key_; }
-  const grpc_slice& value() const { return value_; }
   UserData* user_data() { return &user_data_; }
 
-#ifndef NDEBUG
-  void Ref(const char* file, int line) {
-    grpc_mdelem_trace_ref(this, key_, value_, RefValue(), file, line);
-    Ref();
-  }
-  bool Unref(const char* file, int line) {
-    grpc_mdelem_trace_unref(this, key_, value_, RefValue(), file, line);
-    return Unref();
-  }
-#endif  // ifndef NDEBUG
-  void Ref() {
-    /* we can assume the ref count is >= 1 as the application is calling
-       this function - meaning that no adjustment to mdtab_free is necessary,
-       simplifying the logic here to be just an atomic increment */
-    refcnt_.FetchAdd(1, MemoryOrder::RELAXED);
-  }
-  bool Unref() {
-    const intptr_t prior = refcnt_.FetchSub(1, MemoryOrder::ACQ_REL);
-    GPR_DEBUG_ASSERT(prior > 0);
-    return prior == 1;
-  }
-
  private:
-  intptr_t RefValue() { return refcnt_.Load(MemoryOrder::RELAXED); }
-
-  /* must be byte compatible with grpc_mdelem_data */
-  grpc_slice key_;
-  grpc_slice value_;
-
-  /* private only data */
-  grpc_core::Atomic<intptr_t> refcnt_;
-
   UserData user_data_;
 };
 
@@ -320,30 +366,41 @@ inline grpc_mdelem grpc_mdelem_ref(grpc_mdelem gmd) {
 
 #ifndef NDEBUG
 #define GRPC_MDELEM_UNREF(s) grpc_mdelem_unref((s), __FILE__, __LINE__)
-void grpc_mdelem_do_unref(grpc_mdelem gmd, const char* file, int line);
+void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
+                                uint32_t hash, const char* file, int line);
 inline void grpc_mdelem_unref(grpc_mdelem gmd, const char* file, int line) {
 #else
 #define GRPC_MDELEM_UNREF(s) grpc_mdelem_unref((s))
-void grpc_mdelem_do_unref(grpc_mdelem gmd);
+void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
+                                uint32_t hash);
 inline void grpc_mdelem_unref(grpc_mdelem gmd) {
 #endif
-  switch (GRPC_MDELEM_STORAGE(gmd)) {
+  const grpc_mdelem_data_storage storage = GRPC_MDELEM_STORAGE(gmd);
+  switch (storage) {
     case GRPC_MDELEM_STORAGE_EXTERNAL:
     case GRPC_MDELEM_STORAGE_STATIC:
       return;
     case GRPC_MDELEM_STORAGE_INTERNED:
     case GRPC_MDELEM_STORAGE_ALLOCATED:
+      auto* md =
+          reinterpret_cast<grpc_core::RefcountedMdBase*> GRPC_MDELEM_DATA(gmd);
+      /* once the refcount hits zero, some other thread can come along and
+         free an interned md at any time: it's unsafe from this point on to
+         access it so we read the hash now. */
+      uint32_t hash = md->hash();
 #ifndef NDEBUG
-      grpc_mdelem_do_unref(gmd, file, line);
+      if (GPR_UNLIKELY(md->Unref(file, line))) {
+        grpc_mdelem_on_final_unref(storage, md, hash, file, line);
 #else
-      grpc_mdelem_do_unref(gmd);
+      if (GPR_UNLIKELY(md->Unref())) {
+        grpc_mdelem_on_final_unref(storage, md, hash);
 #endif
+      }
       return;
   }
 }
 
 #define GRPC_MDNULL GRPC_MAKE_MDELEM(NULL, GRPC_MDELEM_STORAGE_EXTERNAL)
-#define GRPC_MDISNULL(md) (GRPC_MDELEM_DATA(md) == NULL)
 
 /* We add 32 bytes of padding as per RFC-7540 section 6.5.2. */
 #define GRPC_MDELEM_LENGTH(e)                                                  \
@@ -355,4 +412,35 @@ inline void grpc_mdelem_unref(grpc_mdelem gmd) {
 void grpc_mdctx_global_init(void);
 void grpc_mdctx_global_shutdown();
 
+/* Like grpc_mdelem_from_slices, but we know that key is a static or interned
+   slice and value is not static or interned. This gives us an inlinable
+   fastpath - we know we must allocate metadata now, and that we do not need to
+   unref the value (rather, we just transfer the ref). We can avoid a ref since:
+   1) the key slice is passed in already ref'd
+   2) We're guaranteed to create a new Allocated slice, thus meaning the
+      ref can be considered 'transferred'.*/
+inline grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::ManagedMemorySlice& key,
+    const grpc_core::UnmanagedMemorySlice& value) {
+  using grpc_core::AllocatedMetadata;
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
+                          GRPC_MDELEM_STORAGE_ALLOCATED);
+}
+
+inline grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::ExternallyManagedSlice& key,
+    const grpc_core::UnmanagedMemorySlice& value) {
+  using grpc_core::AllocatedMetadata;
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
+                          GRPC_MDELEM_STORAGE_ALLOCATED);
+}
+
+inline grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::StaticMetadataSlice& key,
+    const grpc_core::UnmanagedMemorySlice& value) {
+  using grpc_core::AllocatedMetadata;
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
+                          GRPC_MDELEM_STORAGE_ALLOCATED);
+}
+
 #endif /* GRPC_CORE_LIB_TRANSPORT_METADATA_H */

data/src/core/lib/transport/metadata_batch.cc

@@ -50,6 +50,9 @@ static void assert_valid_list(grpc_mdelem_list* list) {
     verified_count++;
   }
   GPR_ASSERT(list->count == verified_count);
+#else
+  // Avoid unused-parameter warning for debug-only parameter
+  (void)list;
 #endif /* NDEBUG */
 }
 
@@ -64,6 +67,9 @@ static void assert_valid_callouts(grpc_metadata_batch* batch) {
     }
     grpc_slice_unref_internal(key_interned);
   }
+#else
+  // Avoid unused-parameter warning for debug-only parameter
+  (void)batch;
 #endif
 }
 
@@ -93,6 +99,23 @@ grpc_error* grpc_attach_md_to_error(grpc_error* src, grpc_mdelem md) {
   return out;
 }
 
+static grpc_error* GPR_ATTRIBUTE_NOINLINE error_with_md(grpc_mdelem md) {
+  return grpc_attach_md_to_error(
+      GRPC_ERROR_CREATE_FROM_STATIC_STRING("Unallowed duplicate metadata"), md);
+}
+
+static grpc_error* link_callout(grpc_metadata_batch* batch,
+                                grpc_linked_mdelem* storage,
+                                grpc_metadata_batch_callouts_index idx) {
+  GPR_DEBUG_ASSERT(idx >= 0 && idx < GRPC_BATCH_CALLOUTS_COUNT);
+  if (GPR_LIKELY(batch->idx.array[idx] == nullptr)) {
+    ++batch->list.default_count;
+    batch->idx.array[idx] = storage;
+    return GRPC_ERROR_NONE;
+  }
+  return error_with_md(storage->md);
+}
+
 static grpc_error* maybe_link_callout(grpc_metadata_batch* batch,
                                       grpc_linked_mdelem* storage)
     GRPC_MUST_USE_RESULT;
@@ -104,14 +127,7 @@ static grpc_error* maybe_link_callout(grpc_metadata_batch* batch,
   if (idx == GRPC_BATCH_CALLOUTS_COUNT) {
     return GRPC_ERROR_NONE;
   }
-  if (batch->idx.array[idx] == nullptr) {
-    ++batch->list.default_count;
-    batch->idx.array[idx] = storage;
-    return GRPC_ERROR_NONE;
-  }
-  return grpc_attach_md_to_error(
-      GRPC_ERROR_CREATE_FROM_STATIC_STRING("Unallowed duplicate metadata"),
-      storage->md);
+  return link_callout(batch, storage, idx);
 }
 
 static void maybe_unlink_callout(grpc_metadata_batch* batch,
@@ -122,21 +138,21 @@ static void maybe_unlink_callout(grpc_metadata_batch* batch,
     return;
   }
   --batch->list.default_count;
-  GPR_ASSERT(batch->idx.array[idx] != nullptr);
+  GPR_DEBUG_ASSERT(batch->idx.array[idx] != nullptr);
   batch->idx.array[idx] = nullptr;
 }
 
 grpc_error* grpc_metadata_batch_add_head(grpc_metadata_batch* batch,
                                          grpc_linked_mdelem* storage,
                                          grpc_mdelem elem_to_add) {
-  GPR_ASSERT(!GRPC_MDISNULL(elem_to_add));
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(elem_to_add));
   storage->md = elem_to_add;
   return grpc_metadata_batch_link_head(batch, storage);
 }
 
 static void link_head(grpc_mdelem_list* list, grpc_linked_mdelem* storage) {
   assert_valid_list(list);
-  GPR_ASSERT(!GRPC_MDISNULL(storage->md));
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(storage->md));
   storage->prev = nullptr;
   storage->next = list->head;
   storage->reserved = nullptr;
@@ -163,17 +179,35 @@ grpc_error* grpc_metadata_batch_link_head(grpc_metadata_batch* batch,
   return GRPC_ERROR_NONE;
 }
 
+// TODO(arjunroy): Need to revisit this and see what guarantees exist between
+// C-core and the internal-metadata subsystem. E.g. can we ensure a particular
+// metadata is never added twice, even in the presence of user supplied data?
+grpc_error* grpc_metadata_batch_link_head(
+    grpc_metadata_batch* batch, grpc_linked_mdelem* storage,
+    grpc_metadata_batch_callouts_index idx) {
+  GPR_DEBUG_ASSERT(GRPC_BATCH_INDEX_OF(GRPC_MDKEY(storage->md)) == idx);
+  assert_valid_callouts(batch);
+  grpc_error* err = link_callout(batch, storage, idx);
+  if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) {
+    assert_valid_callouts(batch);
+    return err;
+  }
+  link_head(&batch->list, storage);
+  assert_valid_callouts(batch);
+  return GRPC_ERROR_NONE;
+}
+
 grpc_error* grpc_metadata_batch_add_tail(grpc_metadata_batch* batch,
                                          grpc_linked_mdelem* storage,
                                          grpc_mdelem elem_to_add) {
-  GPR_ASSERT(!GRPC_MDISNULL(elem_to_add));
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(elem_to_add));
   storage->md = elem_to_add;
   return grpc_metadata_batch_link_tail(batch, storage);
 }
 
 static void link_tail(grpc_mdelem_list* list, grpc_linked_mdelem* storage) {
   assert_valid_list(list);
-  GPR_ASSERT(!GRPC_MDISNULL(storage->md));
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(storage->md));
   storage->prev = list->tail;
   storage->next = nullptr;
   storage->reserved = nullptr;
@@ -200,6 +234,21 @@ grpc_error* grpc_metadata_batch_link_tail(grpc_metadata_batch* batch,
   return GRPC_ERROR_NONE;
 }
 
+grpc_error* grpc_metadata_batch_link_tail(
+    grpc_metadata_batch* batch, grpc_linked_mdelem* storage,
+    grpc_metadata_batch_callouts_index idx) {
+  GPR_DEBUG_ASSERT(GRPC_BATCH_INDEX_OF(GRPC_MDKEY(storage->md)) == idx);
+  assert_valid_callouts(batch);
+  grpc_error* err = link_callout(batch, storage, idx);
+  if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) {
+    assert_valid_callouts(batch);
+    return err;
+  }
+  link_tail(&batch->list, storage);
+  assert_valid_callouts(batch);
+  return GRPC_ERROR_NONE;
+}
+
 static void unlink_storage(grpc_mdelem_list* list,
                            grpc_linked_mdelem* storage) {
   assert_valid_list(list);
@@ -226,6 +275,18 @@ void grpc_metadata_batch_remove(grpc_metadata_batch* batch,
   assert_valid_callouts(batch);
 }
 
+void grpc_metadata_batch_remove(grpc_metadata_batch* batch,
+                                grpc_metadata_batch_callouts_index idx) {
+  assert_valid_callouts(batch);
+  grpc_linked_mdelem* storage = batch->idx.array[idx];
+  GPR_DEBUG_ASSERT(storage != nullptr);
+  --batch->list.default_count;
+  batch->idx.array[idx] = nullptr;
+  unlink_storage(&batch->list, storage);
+  GRPC_MDELEM_UNREF(storage->md);
+  assert_valid_callouts(batch);
+}
+
 void grpc_metadata_batch_set_value(grpc_linked_mdelem* storage,
                                    const grpc_slice& value) {
   grpc_mdelem old_mdelem = storage->md;
@@ -313,13 +374,14 @@ void grpc_metadata_batch_copy(grpc_metadata_batch* src,
   size_t i = 0;
   for (grpc_linked_mdelem* elem = src->list.head; elem != nullptr;
        elem = elem->next) {
-    grpc_error* error = grpc_metadata_batch_add_tail(dst, &storage[i++],
-                                                     GRPC_MDELEM_REF(elem->md));
+    // Error unused in non-debug builds.
+    grpc_error* GRPC_UNUSED error = grpc_metadata_batch_add_tail(
+        dst, &storage[i++], GRPC_MDELEM_REF(elem->md));
     // The only way that grpc_metadata_batch_add_tail() can fail is if
     // there's a duplicate entry for a callout.  However, that can't be
     // the case here, because we would not have been allowed to create
     // a source batch that had that kind of conflict.
-    GPR_ASSERT(error == GRPC_ERROR_NONE);
+    GPR_DEBUG_ASSERT(error == GRPC_ERROR_NONE);
   }
 }