grpc 1.30.2

data/src/core/lib/transport/metadata.h

@@ -0,0 +1,446 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_TRANSPORT_METADATA_H
+#define GRPC_CORE_LIB_TRANSPORT_METADATA_H
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/impl/codegen/log.h>
+
+#include <grpc/grpc.h>
+#include <grpc/slice.h>
+
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/gprpp/atomic.h"
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/slice/slice_utils.h"
+
+extern grpc_core::DebugOnlyTraceFlag grpc_trace_metadata;
+
+/* This file provides a mechanism for tracking metadata through the grpc stack.
+   It's not intended for consumption outside of the library.
+
+   Metadata is tracked in the context of a sharded global grpc_mdctx. The
+   context tracks unique strings (grpc_mdstr) and pairs of strings
+   (grpc_mdelem). Any of these objects can be checked for equality by comparing
+   their pointers. These objects are reference counted.
+
+   grpc_mdelem can additionally store a (non-NULL) user data pointer. This
+   pointer is intended to be used to cache semantic meaning of a metadata
+   element. For example, an OAuth token may cache the credentials it represents
+   and the time at which it expires in the mdelem user data.
+
+   Combining this metadata cache and the hpack compression table allows us to
+   simply lookup complete preparsed objects quickly, incurring a few atomic
+   ops per metadata element on the fast path.
+
+   grpc_mdelem instances MAY live longer than their refcount implies, and are
+   garbage collected periodically, meaning cached data can easily outlive a
+   single request.
+
+   STATIC METADATA: in static_metadata.h we declare a set of static metadata.
+   These mdelems and mdstrs are available via pre-declared code generated macros
+   and are available to code anywhere between grpc_init() and grpc_shutdown().
+   They are not refcounted, but can be passed to _ref and _unref functions
+   declared here - in which case those functions are effectively no-ops. */
+
+/* Forward declarations */
+typedef struct grpc_mdelem grpc_mdelem;
+
+/* if changing this, make identical changes in:
+   - grpc_core::{InternedMetadata, AllocatedMetadata}
+   - grpc_metadata in grpc_types.h */
+typedef struct grpc_mdelem_data {
+  const grpc_slice key;
+  const grpc_slice value;
+  /* there is a private part to this in metadata.c */
+} grpc_mdelem_data;
+
+/* GRPC_MDELEM_STORAGE_* enum values that can be treated as interned always have
+   this bit set in their integer value */
+#define GRPC_MDELEM_STORAGE_INTERNED_BIT 1
+
+/* External and static storage metadata has no refcount to ref/unref. Allocated
+ * and interned metadata do have a refcount. Metadata ref and unref methods use
+ * a switch statement on this enum to determine which behaviour to execute.
+ * Keeping the no-ref cases together and the ref-cases together leads to
+ * slightly better code generation (9 inlined instructions rather than 10). */
+typedef enum {
+  /* memory pointed to by grpc_mdelem::payload is owned by an external system */
+  GRPC_MDELEM_STORAGE_EXTERNAL = 0,
+  /* memory is in the static metadata table */
+  GRPC_MDELEM_STORAGE_STATIC = GRPC_MDELEM_STORAGE_INTERNED_BIT,
+  /* memory pointed to by grpc_mdelem::payload is allocated by the metadata
+     system */
+  GRPC_MDELEM_STORAGE_ALLOCATED = 2,
+  /* memory pointed to by grpc_mdelem::payload is interned by the metadata
+     system */
+  GRPC_MDELEM_STORAGE_INTERNED = 2 | GRPC_MDELEM_STORAGE_INTERNED_BIT,
+} grpc_mdelem_data_storage;
+
+struct grpc_mdelem {
+  /* a grpc_mdelem_data* generally, with the two lower bits signalling memory
+     ownership as per grpc_mdelem_data_storage */
+  uintptr_t payload;
+};
+
+#define GRPC_MDELEM_DATA(md) ((grpc_mdelem_data*)((md).payload & ~(uintptr_t)3))
+#define GRPC_MDELEM_STORAGE(md) \
+  ((grpc_mdelem_data_storage)((md).payload & (uintptr_t)3))
+#ifdef __cplusplus
+#define GRPC_MAKE_MDELEM(data, storage) \
+  (grpc_mdelem{((uintptr_t)(data)) | ((uintptr_t)storage)})
+#else
+#define GRPC_MAKE_MDELEM(data, storage) \
+  ((grpc_mdelem){((uintptr_t)(data)) | ((uintptr_t)storage)})
+#endif
+#define GRPC_MDELEM_IS_INTERNED(md)          \
+  ((grpc_mdelem_data_storage)((md).payload & \
+                              (uintptr_t)GRPC_MDELEM_STORAGE_INTERNED_BIT))
+
+/* Given arbitrary input slices, create a grpc_mdelem object. The caller refs
+ * the input slices; we unref them. This method is always safe to call; however,
+ * if we know data about the slices in question (e.g. if we knew our key was
+ * static) we can call specializations that save on cycle count. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_slice& key,
+                                    const grpc_slice& value);
+
+/* Like grpc_mdelem_from_slices, but we know that key is a static slice. This
+   saves us a few branches and a no-op call to md_unref() for the key. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_core::StaticMetadataSlice& key,
+                                    const grpc_slice& value);
+
+/* Like grpc_mdelem_from_slices, but key is static and val is static. */
+grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::StaticMetadataSlice& key,
+    const grpc_core::StaticMetadataSlice& value);
+
+/* Like grpc_mdelem_from_slices, but key is static and val is interned. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_core::StaticMetadataSlice& key,
+                                    const grpc_core::ManagedMemorySlice& value);
+
+/* Like grpc_mdelem_from_slices, but key and val are interned. */
+grpc_mdelem grpc_mdelem_from_slices(const grpc_core::ManagedMemorySlice& key,
+                                    const grpc_core::ManagedMemorySlice& value);
+
+/* Cheaply convert a grpc_metadata to a grpc_mdelem; may use the grpc_metadata
+   object as backing storage (so lifetimes should align) */
+grpc_mdelem grpc_mdelem_from_grpc_metadata(grpc_metadata* metadata);
+
+/* Does not unref the slices; if a new non-interned mdelem is needed, allocates
+   one if compatible_external_backing_store is NULL, or uses
+   compatible_external_backing_store if it is non-NULL (in which case it's the
+   users responsibility to ensure that it outlives usage) */
+grpc_mdelem grpc_mdelem_create(
+    const grpc_slice& key, const grpc_slice& value,
+    grpc_mdelem_data* compatible_external_backing_store);
+
+/* Like grpc_mdelem_create, but we know that key is static. */
+grpc_mdelem grpc_mdelem_create(
+    const grpc_core::StaticMetadataSlice& key, const grpc_slice& value,
+    grpc_mdelem_data* compatible_external_backing_store);
+
+#define GRPC_MDKEY(md) (GRPC_MDELEM_DATA(md)->key)
+#define GRPC_MDVALUE(md) (GRPC_MDELEM_DATA(md)->value)
+
+bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b);
+/* Often we compare metadata where we know a-priori that the second parameter is
+ * static, and that the keys match. This most commonly happens when processing
+ * metadata batch callouts in initial/trailing filters. In this case, fastpath
+ * grpc_mdelem_eq and remove unnecessary checks. */
+inline bool grpc_mdelem_static_value_eq(grpc_mdelem a, grpc_mdelem b_static) {
+  if (a.payload == b_static.payload) return true;
+  return grpc_slice_eq_static_interned(GRPC_MDVALUE(a), GRPC_MDVALUE(b_static));
+}
+#define GRPC_MDISNULL(md) (GRPC_MDELEM_DATA(md) == NULL)
+
+inline bool grpc_mdelem_both_interned_eq(grpc_mdelem a_interned,
+                                         grpc_mdelem b_interned) {
+  GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(a_interned) ||
+                   GRPC_MDISNULL(a_interned));
+  GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(b_interned) ||
+                   GRPC_MDISNULL(b_interned));
+  return a_interned.payload == b_interned.payload;
+}
+
+/* Mutator and accessor for grpc_mdelem user data. The destructor function
+   is used as a type tag and is checked during user_data fetch. */
+void* grpc_mdelem_get_user_data(grpc_mdelem md, void (*if_destroy_func)(void*));
+void* grpc_mdelem_set_user_data(grpc_mdelem md, void (*destroy_func)(void*),
+                                void* data);
+
+// Defined in metadata.cc.
+struct mdtab_shard;
+
+#ifndef NDEBUG
+void grpc_mdelem_trace_ref(void* md, const grpc_slice& key,
+                           const grpc_slice& value, intptr_t refcnt,
+                           const char* file, int line);
+void grpc_mdelem_trace_unref(void* md, const grpc_slice& key,
+                             const grpc_slice& value, intptr_t refcnt,
+                             const char* file, int line);
+#endif
+namespace grpc_core {
+
+typedef void (*destroy_user_data_func)(void* data);
+
+struct UserData {
+  Mutex mu_user_data;
+  grpc_core::Atomic<destroy_user_data_func> destroy_user_data;
+  grpc_core::Atomic<void*> data;
+};
+
+class StaticMetadata {
+ public:
+  StaticMetadata(const grpc_slice& key, const grpc_slice& value, uintptr_t idx)
+      : kv_({key, value}), hash_(0), static_idx_(idx) {}
+
+  const grpc_mdelem_data& data() const { return kv_; }
+
+  void HashInit();
+  uint32_t hash() { return hash_; }
+  uintptr_t StaticIndex() { return static_idx_; }
+
+ private:
+  grpc_mdelem_data kv_;
+
+  /* private only data */
+  uint32_t hash_;
+  uintptr_t static_idx_;
+};
+
+class RefcountedMdBase {
+ public:
+  RefcountedMdBase(const grpc_slice& key, const grpc_slice& value)
+      : key_(key), value_(value), refcnt_(1) {}
+  RefcountedMdBase(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash)
+      : key_(key), value_(value), refcnt_(1), hash_(hash) {}
+
+  const grpc_slice& key() const { return key_; }
+  const grpc_slice& value() const { return value_; }
+  uint32_t hash() { return hash_; }
+
+#ifndef NDEBUG
+  void Ref(const char* file, int line) {
+    grpc_mdelem_trace_ref(this, key_, value_, RefValue(), file, line);
+    const intptr_t prior = refcnt_.FetchAdd(1, MemoryOrder::RELAXED);
+    GPR_ASSERT(prior > 0);
+  }
+  bool Unref(const char* file, int line) {
+    grpc_mdelem_trace_unref(this, key_, value_, RefValue(), file, line);
+    return Unref();
+  }
+#endif
+  void Ref() {
+    /* we can assume the ref count is >= 1 as the application is calling
+       this function - meaning that no adjustment to mdtab_free is necessary,
+       simplifying the logic here to be just an atomic increment */
+    refcnt_.FetchAdd(1, MemoryOrder::RELAXED);
+  }
+  bool Unref() {
+    const intptr_t prior = refcnt_.FetchSub(1, MemoryOrder::ACQ_REL);
+    GPR_DEBUG_ASSERT(prior > 0);
+    return prior == 1;
+  }
+
+ protected:
+#ifndef NDEBUG
+  void TraceAtStart(const char* tag);
+#endif
+
+  intptr_t RefValue() { return refcnt_.Load(MemoryOrder::RELAXED); }
+  bool AllRefsDropped() { return refcnt_.Load(MemoryOrder::ACQUIRE) == 0; }
+  bool FirstRef() { return refcnt_.FetchAdd(1, MemoryOrder::RELAXED) == 0; }
+
+ private:
+  /* must be byte compatible with grpc_mdelem_data */
+  grpc_slice key_;
+  grpc_slice value_;
+  grpc_core::Atomic<intptr_t> refcnt_;
+  uint32_t hash_ = 0;
+};
+
+class InternedMetadata : public RefcountedMdBase {
+ public:
+  // TODO(arjunroy): Change to use strongly typed slices instead.
+  struct NoRefKey {};
+  struct BucketLink {
+    explicit BucketLink(InternedMetadata* md) : next(md) {}
+
+    InternedMetadata* next = nullptr;
+  };
+  InternedMetadata(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash, InternedMetadata* next);
+  InternedMetadata(const grpc_slice& key, const grpc_slice& value,
+                   uint32_t hash, InternedMetadata* next, const NoRefKey*);
+
+  ~InternedMetadata();
+  void RefWithShardLocked(mdtab_shard* shard);
+  UserData* user_data() { return &user_data_; }
+  InternedMetadata* bucket_next() { return link_.next; }
+  void set_bucket_next(InternedMetadata* md) { link_.next = md; }
+
+  static size_t CleanupLinkedMetadata(BucketLink* head);
+
+ private:
+  UserData user_data_;
+  BucketLink link_;
+};
+
+/* Shadow structure for grpc_mdelem_data for allocated elements */
+class AllocatedMetadata : public RefcountedMdBase {
+ public:
+  // TODO(arjunroy): Change to use strongly typed slices instead.
+  struct NoRefKey {};
+  AllocatedMetadata(const grpc_slice& key, const grpc_slice& value);
+  AllocatedMetadata(const grpc_core::ManagedMemorySlice& key,
+                    const grpc_core::UnmanagedMemorySlice& value);
+  AllocatedMetadata(const grpc_core::ExternallyManagedSlice& key,
+                    const grpc_core::UnmanagedMemorySlice& value);
+  AllocatedMetadata(const grpc_slice& key, const grpc_slice& value,
+                    const NoRefKey*);
+  ~AllocatedMetadata();
+
+  UserData* user_data() { return &user_data_; }
+
+ private:
+  UserData user_data_;
+};
+
+}  // namespace grpc_core
+
+#ifndef NDEBUG
+#define GRPC_MDELEM_REF(s) grpc_mdelem_ref((s), __FILE__, __LINE__)
+inline grpc_mdelem grpc_mdelem_ref(grpc_mdelem gmd, const char* file,
+                                   int line) {
+#else  // ifndef NDEBUG
+#define GRPC_MDELEM_REF(s) grpc_mdelem_ref((s))
+inline grpc_mdelem grpc_mdelem_ref(grpc_mdelem gmd) {
+#endif  // ifndef NDEBUG
+  switch (GRPC_MDELEM_STORAGE(gmd)) {
+    case GRPC_MDELEM_STORAGE_EXTERNAL:
+    case GRPC_MDELEM_STORAGE_STATIC:
+      break;
+    case GRPC_MDELEM_STORAGE_INTERNED: {
+      auto* md =
+          reinterpret_cast<grpc_core::InternedMetadata*> GRPC_MDELEM_DATA(gmd);
+      /* use C assert to have this removed in opt builds */
+#ifndef NDEBUG
+      md->Ref(file, line);
+#else
+      md->Ref();
+#endif
+      break;
+    }
+    case GRPC_MDELEM_STORAGE_ALLOCATED: {
+      auto* md =
+          reinterpret_cast<grpc_core::AllocatedMetadata*> GRPC_MDELEM_DATA(gmd);
+#ifndef NDEBUG
+      md->Ref(file, line);
+#else
+      md->Ref();
+#endif
+      break;
+    }
+  }
+  return gmd;
+}
+
+#ifndef NDEBUG
+#define GRPC_MDELEM_UNREF(s) grpc_mdelem_unref((s), __FILE__, __LINE__)
+void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
+                                uint32_t hash, const char* file, int line);
+inline void grpc_mdelem_unref(grpc_mdelem gmd, const char* file, int line) {
+#else
+#define GRPC_MDELEM_UNREF(s) grpc_mdelem_unref((s))
+void grpc_mdelem_on_final_unref(grpc_mdelem_data_storage storage, void* ptr,
+                                uint32_t hash);
+inline void grpc_mdelem_unref(grpc_mdelem gmd) {
+#endif
+  const grpc_mdelem_data_storage storage = GRPC_MDELEM_STORAGE(gmd);
+  switch (storage) {
+    case GRPC_MDELEM_STORAGE_EXTERNAL:
+    case GRPC_MDELEM_STORAGE_STATIC:
+      return;
+    case GRPC_MDELEM_STORAGE_INTERNED:
+    case GRPC_MDELEM_STORAGE_ALLOCATED:
+      auto* md =
+          reinterpret_cast<grpc_core::RefcountedMdBase*> GRPC_MDELEM_DATA(gmd);
+      /* once the refcount hits zero, some other thread can come along and
+         free an interned md at any time: it's unsafe from this point on to
+         access it so we read the hash now. */
+      uint32_t hash = md->hash();
+#ifndef NDEBUG
+      if (GPR_UNLIKELY(md->Unref(file, line))) {
+        grpc_mdelem_on_final_unref(storage, md, hash, file, line);
+#else
+      if (GPR_UNLIKELY(md->Unref())) {
+        grpc_mdelem_on_final_unref(storage, md, hash);
+#endif
+      }
+      return;
+  }
+}
+
+#define GRPC_MDNULL GRPC_MAKE_MDELEM(NULL, GRPC_MDELEM_STORAGE_EXTERNAL)
+
+/* We add 32 bytes of padding as per RFC-7540 section 6.5.2. */
+#define GRPC_MDELEM_LENGTH(e)                                                  \
+  (GRPC_SLICE_LENGTH(GRPC_MDKEY((e))) + GRPC_SLICE_LENGTH(GRPC_MDVALUE((e))) + \
+   32)
+
+#define GRPC_MDSTR_KV_HASH(k_hash, v_hash) (GPR_ROTL((k_hash), 2) ^ (v_hash))
+
+void grpc_mdctx_global_init(void);
+void grpc_mdctx_global_shutdown();
+
+/* Like grpc_mdelem_from_slices, but we know that key is a static or interned
+   slice and value is not static or interned. This gives us an inlinable
+   fastpath - we know we must allocate metadata now, and that we do not need to
+   unref the value (rather, we just transfer the ref). We can avoid a ref since:
+   1) the key slice is passed in already ref'd
+   2) We're guaranteed to create a new Allocated slice, thus meaning the
+      ref can be considered 'transferred'.*/
+inline grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::ManagedMemorySlice& key,
+    const grpc_core::UnmanagedMemorySlice& value) {
+  using grpc_core::AllocatedMetadata;
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
+                          GRPC_MDELEM_STORAGE_ALLOCATED);
+}
+
+inline grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::ExternallyManagedSlice& key,
+    const grpc_core::UnmanagedMemorySlice& value) {
+  using grpc_core::AllocatedMetadata;
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
+                          GRPC_MDELEM_STORAGE_ALLOCATED);
+}
+
+inline grpc_mdelem grpc_mdelem_from_slices(
+    const grpc_core::StaticMetadataSlice& key,
+    const grpc_core::UnmanagedMemorySlice& value) {
+  using grpc_core::AllocatedMetadata;
+  return GRPC_MAKE_MDELEM(new AllocatedMetadata(key, value),
+                          GRPC_MDELEM_STORAGE_ALLOCATED);
+}
+
+#endif /* GRPC_CORE_LIB_TRANSPORT_METADATA_H */

data/src/core/lib/transport/metadata_batch.cc

@@ -0,0 +1,392 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/transport/metadata_batch.h"
+
+#include <stdbool.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/slice/slice_string_helpers.h"
+
+static void assert_valid_list(grpc_mdelem_list* list) {
+#ifndef NDEBUG
+  grpc_linked_mdelem* l;
+
+  GPR_ASSERT((list->head == nullptr) == (list->tail == nullptr));
+  if (!list->head) return;
+  GPR_ASSERT(list->head->prev == nullptr);
+  GPR_ASSERT(list->tail->next == nullptr);
+  GPR_ASSERT((list->head == list->tail) == (list->head->next == nullptr));
+
+  size_t verified_count = 0;
+  for (l = list->head; l; l = l->next) {
+    GPR_ASSERT(!GRPC_MDISNULL(l->md));
+    GPR_ASSERT((l->prev == nullptr) == (l == list->head));
+    GPR_ASSERT((l->next == nullptr) == (l == list->tail));
+    if (l->next) GPR_ASSERT(l->next->prev == l);
+    if (l->prev) GPR_ASSERT(l->prev->next == l);
+    verified_count++;
+  }
+  GPR_ASSERT(list->count == verified_count);
+#else
+  // Avoid unused-parameter warning for debug-only parameter
+  (void)list;
+#endif /* NDEBUG */
+}
+
+static void assert_valid_callouts(grpc_metadata_batch* batch) {
+#ifndef NDEBUG
+  for (grpc_linked_mdelem* l = batch->list.head; l != nullptr; l = l->next) {
+    grpc_slice key_interned = grpc_slice_intern(GRPC_MDKEY(l->md));
+    grpc_metadata_batch_callouts_index callout_idx =
+        GRPC_BATCH_INDEX_OF(key_interned);
+    if (callout_idx != GRPC_BATCH_CALLOUTS_COUNT) {
+      GPR_ASSERT(batch->idx.array[callout_idx] == l);
+    }
+    grpc_slice_unref_internal(key_interned);
+  }
+#else
+  // Avoid unused-parameter warning for debug-only parameter
+  (void)batch;
+#endif
+}
+
+#ifndef NDEBUG
+void grpc_metadata_batch_assert_ok(grpc_metadata_batch* batch) {
+  assert_valid_list(&batch->list);
+}
+#endif /* NDEBUG */
+
+void grpc_metadata_batch_init(grpc_metadata_batch* batch) {
+  memset(batch, 0, sizeof(*batch));
+  batch->deadline = GRPC_MILLIS_INF_FUTURE;
+}
+
+void grpc_metadata_batch_destroy(grpc_metadata_batch* batch) {
+  grpc_linked_mdelem* l;
+  for (l = batch->list.head; l; l = l->next) {
+    GRPC_MDELEM_UNREF(l->md);
+  }
+}
+
+grpc_error* grpc_attach_md_to_error(grpc_error* src, grpc_mdelem md) {
+  grpc_error* out = grpc_error_set_str(
+      grpc_error_set_str(src, GRPC_ERROR_STR_KEY,
+                         grpc_slice_ref_internal(GRPC_MDKEY(md))),
+      GRPC_ERROR_STR_VALUE, grpc_slice_ref_internal(GRPC_MDVALUE(md)));
+  return out;
+}
+
+static grpc_error* GPR_ATTRIBUTE_NOINLINE error_with_md(grpc_mdelem md) {
+  return grpc_attach_md_to_error(
+      GRPC_ERROR_CREATE_FROM_STATIC_STRING("Unallowed duplicate metadata"), md);
+}
+
+static grpc_error* link_callout(grpc_metadata_batch* batch,
+                                grpc_linked_mdelem* storage,
+                                grpc_metadata_batch_callouts_index idx) {
+  GPR_DEBUG_ASSERT(idx >= 0 && idx < GRPC_BATCH_CALLOUTS_COUNT);
+  if (GPR_LIKELY(batch->idx.array[idx] == nullptr)) {
+    ++batch->list.default_count;
+    batch->idx.array[idx] = storage;
+    return GRPC_ERROR_NONE;
+  }
+  return error_with_md(storage->md);
+}
+
+static grpc_error* maybe_link_callout(grpc_metadata_batch* batch,
+                                      grpc_linked_mdelem* storage)
+    GRPC_MUST_USE_RESULT;
+
+static grpc_error* maybe_link_callout(grpc_metadata_batch* batch,
+                                      grpc_linked_mdelem* storage) {
+  grpc_metadata_batch_callouts_index idx =
+      GRPC_BATCH_INDEX_OF(GRPC_MDKEY(storage->md));
+  if (idx == GRPC_BATCH_CALLOUTS_COUNT) {
+    return GRPC_ERROR_NONE;
+  }
+  return link_callout(batch, storage, idx);
+}
+
+static void maybe_unlink_callout(grpc_metadata_batch* batch,
+                                 grpc_linked_mdelem* storage) {
+  grpc_metadata_batch_callouts_index idx =
+      GRPC_BATCH_INDEX_OF(GRPC_MDKEY(storage->md));
+  if (idx == GRPC_BATCH_CALLOUTS_COUNT) {
+    return;
+  }
+  --batch->list.default_count;
+  GPR_DEBUG_ASSERT(batch->idx.array[idx] != nullptr);
+  batch->idx.array[idx] = nullptr;
+}
+
+grpc_error* grpc_metadata_batch_add_head(grpc_metadata_batch* batch,
+                                         grpc_linked_mdelem* storage,
+                                         grpc_mdelem elem_to_add) {
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(elem_to_add));
+  storage->md = elem_to_add;
+  return grpc_metadata_batch_link_head(batch, storage);
+}
+
+static void link_head(grpc_mdelem_list* list, grpc_linked_mdelem* storage) {
+  assert_valid_list(list);
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(storage->md));
+  storage->prev = nullptr;
+  storage->next = list->head;
+  storage->reserved = nullptr;
+  if (list->head != nullptr) {
+    list->head->prev = storage;
+  } else {
+    list->tail = storage;
+  }
+  list->head = storage;
+  list->count++;
+  assert_valid_list(list);
+}
+
+grpc_error* grpc_metadata_batch_link_head(grpc_metadata_batch* batch,
+                                          grpc_linked_mdelem* storage) {
+  assert_valid_callouts(batch);
+  grpc_error* err = maybe_link_callout(batch, storage);
+  if (err != GRPC_ERROR_NONE) {
+    assert_valid_callouts(batch);
+    return err;
+  }
+  link_head(&batch->list, storage);
+  assert_valid_callouts(batch);
+  return GRPC_ERROR_NONE;
+}
+
+// TODO(arjunroy): Need to revisit this and see what guarantees exist between
+// C-core and the internal-metadata subsystem. E.g. can we ensure a particular
+// metadata is never added twice, even in the presence of user supplied data?
+grpc_error* grpc_metadata_batch_link_head(
+    grpc_metadata_batch* batch, grpc_linked_mdelem* storage,
+    grpc_metadata_batch_callouts_index idx) {
+  GPR_DEBUG_ASSERT(GRPC_BATCH_INDEX_OF(GRPC_MDKEY(storage->md)) == idx);
+  assert_valid_callouts(batch);
+  grpc_error* err = link_callout(batch, storage, idx);
+  if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) {
+    assert_valid_callouts(batch);
+    return err;
+  }
+  link_head(&batch->list, storage);
+  assert_valid_callouts(batch);
+  return GRPC_ERROR_NONE;
+}
+
+grpc_error* grpc_metadata_batch_add_tail(grpc_metadata_batch* batch,
+                                         grpc_linked_mdelem* storage,
+                                         grpc_mdelem elem_to_add) {
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(elem_to_add));
+  storage->md = elem_to_add;
+  return grpc_metadata_batch_link_tail(batch, storage);
+}
+
+static void link_tail(grpc_mdelem_list* list, grpc_linked_mdelem* storage) {
+  assert_valid_list(list);
+  GPR_DEBUG_ASSERT(!GRPC_MDISNULL(storage->md));
+  storage->prev = list->tail;
+  storage->next = nullptr;
+  storage->reserved = nullptr;
+  if (list->tail != nullptr) {
+    list->tail->next = storage;
+  } else {
+    list->head = storage;
+  }
+  list->tail = storage;
+  list->count++;
+  assert_valid_list(list);
+}
+
+grpc_error* grpc_metadata_batch_link_tail(grpc_metadata_batch* batch,
+                                          grpc_linked_mdelem* storage) {
+  assert_valid_callouts(batch);
+  grpc_error* err = maybe_link_callout(batch, storage);
+  if (err != GRPC_ERROR_NONE) {
+    assert_valid_callouts(batch);
+    return err;
+  }
+  link_tail(&batch->list, storage);
+  assert_valid_callouts(batch);
+  return GRPC_ERROR_NONE;
+}
+
+grpc_error* grpc_metadata_batch_link_tail(
+    grpc_metadata_batch* batch, grpc_linked_mdelem* storage,
+    grpc_metadata_batch_callouts_index idx) {
+  GPR_DEBUG_ASSERT(GRPC_BATCH_INDEX_OF(GRPC_MDKEY(storage->md)) == idx);
+  assert_valid_callouts(batch);
+  grpc_error* err = link_callout(batch, storage, idx);
+  if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) {
+    assert_valid_callouts(batch);
+    return err;
+  }
+  link_tail(&batch->list, storage);
+  assert_valid_callouts(batch);
+  return GRPC_ERROR_NONE;
+}
+
+static void unlink_storage(grpc_mdelem_list* list,
+                           grpc_linked_mdelem* storage) {
+  assert_valid_list(list);
+  if (storage->prev != nullptr) {
+    storage->prev->next = storage->next;
+  } else {
+    list->head = storage->next;
+  }
+  if (storage->next != nullptr) {
+    storage->next->prev = storage->prev;
+  } else {
+    list->tail = storage->prev;
+  }
+  list->count--;
+  assert_valid_list(list);
+}
+
+void grpc_metadata_batch_remove(grpc_metadata_batch* batch,
+                                grpc_linked_mdelem* storage) {
+  assert_valid_callouts(batch);
+  maybe_unlink_callout(batch, storage);
+  unlink_storage(&batch->list, storage);
+  GRPC_MDELEM_UNREF(storage->md);
+  assert_valid_callouts(batch);
+}
+
+void grpc_metadata_batch_remove(grpc_metadata_batch* batch,
+                                grpc_metadata_batch_callouts_index idx) {
+  assert_valid_callouts(batch);
+  grpc_linked_mdelem* storage = batch->idx.array[idx];
+  GPR_DEBUG_ASSERT(storage != nullptr);
+  --batch->list.default_count;
+  batch->idx.array[idx] = nullptr;
+  unlink_storage(&batch->list, storage);
+  GRPC_MDELEM_UNREF(storage->md);
+  assert_valid_callouts(batch);
+}
+
+void grpc_metadata_batch_set_value(grpc_linked_mdelem* storage,
+                                   const grpc_slice& value) {
+  grpc_mdelem old_mdelem = storage->md;
+  grpc_mdelem new_mdelem = grpc_mdelem_from_slices(
+      grpc_slice_ref_internal(GRPC_MDKEY(old_mdelem)), value);
+  storage->md = new_mdelem;
+  GRPC_MDELEM_UNREF(old_mdelem);
+}
+
+grpc_error* grpc_metadata_batch_substitute(grpc_metadata_batch* batch,
+                                           grpc_linked_mdelem* storage,
+                                           grpc_mdelem new_mdelem) {
+  assert_valid_callouts(batch);
+  grpc_error* error = GRPC_ERROR_NONE;
+  grpc_mdelem old_mdelem = storage->md;
+  if (!grpc_slice_eq(GRPC_MDKEY(new_mdelem), GRPC_MDKEY(old_mdelem))) {
+    maybe_unlink_callout(batch, storage);
+    storage->md = new_mdelem;
+    error = maybe_link_callout(batch, storage);
+    if (error != GRPC_ERROR_NONE) {
+      unlink_storage(&batch->list, storage);
+      GRPC_MDELEM_UNREF(storage->md);
+    }
+  } else {
+    storage->md = new_mdelem;
+  }
+  GRPC_MDELEM_UNREF(old_mdelem);
+  assert_valid_callouts(batch);
+  return error;
+}
+
+void grpc_metadata_batch_clear(grpc_metadata_batch* batch) {
+  grpc_metadata_batch_destroy(batch);
+  grpc_metadata_batch_init(batch);
+}
+
+bool grpc_metadata_batch_is_empty(grpc_metadata_batch* batch) {
+  return batch->list.head == nullptr &&
+         batch->deadline == GRPC_MILLIS_INF_FUTURE;
+}
+
+size_t grpc_metadata_batch_size(grpc_metadata_batch* batch) {
+  size_t size = 0;
+  for (grpc_linked_mdelem* elem = batch->list.head; elem != nullptr;
+       elem = elem->next) {
+    size += GRPC_MDELEM_LENGTH(elem->md);
+  }
+  return size;
+}
+
+static void add_error(grpc_error** composite, grpc_error* error,
+                      const char* composite_error_string) {
+  if (error == GRPC_ERROR_NONE) return;
+  if (*composite == GRPC_ERROR_NONE) {
+    *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(composite_error_string);
+  }
+  *composite = grpc_error_add_child(*composite, error);
+}
+
+grpc_error* grpc_metadata_batch_filter(grpc_metadata_batch* batch,
+                                       grpc_metadata_batch_filter_func func,
+                                       void* user_data,
+                                       const char* composite_error_string) {
+  grpc_linked_mdelem* l = batch->list.head;
+  grpc_error* error = GRPC_ERROR_NONE;
+  while (l) {
+    grpc_linked_mdelem* next = l->next;
+    grpc_filtered_mdelem new_mdelem = func(user_data, l->md);
+    add_error(&error, new_mdelem.error, composite_error_string);
+    if (GRPC_MDISNULL(new_mdelem.md)) {
+      grpc_metadata_batch_remove(batch, l);
+    } else if (new_mdelem.md.payload != l->md.payload) {
+      grpc_metadata_batch_substitute(batch, l, new_mdelem.md);
+    }
+    l = next;
+  }
+  return error;
+}
+
+void grpc_metadata_batch_copy(grpc_metadata_batch* src,
+                              grpc_metadata_batch* dst,
+                              grpc_linked_mdelem* storage) {
+  grpc_metadata_batch_init(dst);
+  dst->deadline = src->deadline;
+  size_t i = 0;
+  for (grpc_linked_mdelem* elem = src->list.head; elem != nullptr;
+       elem = elem->next) {
+    // Error unused in non-debug builds.
+    grpc_error* GRPC_UNUSED error = grpc_metadata_batch_add_tail(
+        dst, &storage[i++], GRPC_MDELEM_REF(elem->md));
+    // The only way that grpc_metadata_batch_add_tail() can fail is if
+    // there's a duplicate entry for a callout.  However, that can't be
+    // the case here, because we would not have been allowed to create
+    // a source batch that had that kind of conflict.
+    GPR_DEBUG_ASSERT(error == GRPC_ERROR_NONE);
+  }
+}
+
+void grpc_metadata_batch_move(grpc_metadata_batch* src,
+                              grpc_metadata_batch* dst) {
+  *dst = *src;
+  grpc_metadata_batch_init(src);
+}