grpc 1.40.0 → 1.43.1

data/third_party/abseil-cpp/absl/strings/cord.cc

@@ -36,8 +36,11 @@
 #include "absl/container/inlined_vector.h"
 #include "absl/strings/escaping.h"
 #include "absl/strings/internal/cord_internal.h"
+#include "absl/strings/internal/cord_rep_btree.h"
 #include "absl/strings/internal/cord_rep_flat.h"
-#include "absl/strings/internal/cord_rep_ring.h"
+#include "absl/strings/internal/cordz_statistics.h"
+#include "absl/strings/internal/cordz_update_scope.h"
+#include "absl/strings/internal/cordz_update_tracker.h"
 #include "absl/strings/internal/resize_uninitialized.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
@@ -48,19 +51,15 @@ namespace absl {
 ABSL_NAMESPACE_BEGIN
 
 using ::absl::cord_internal::CordRep;
+using ::absl::cord_internal::CordRepBtree;
 using ::absl::cord_internal::CordRepConcat;
 using ::absl::cord_internal::CordRepExternal;
 using ::absl::cord_internal::CordRepFlat;
-using ::absl::cord_internal::CordRepRing;
 using ::absl::cord_internal::CordRepSubstring;
-using ::absl::cord_internal::kMinFlatLength;
+using ::absl::cord_internal::CordzUpdateTracker;
+using ::absl::cord_internal::InlineData;
 using ::absl::cord_internal::kMaxFlatLength;
-
-using ::absl::cord_internal::CONCAT;
-using ::absl::cord_internal::EXTERNAL;
-using ::absl::cord_internal::FLAT;
-using ::absl::cord_internal::RING;
-using ::absl::cord_internal::SUBSTRING;
+using ::absl::cord_internal::kMinFlatLength;
 
 using ::absl::cord_internal::kInlinedVectorSize;
 using ::absl::cord_internal::kMaxBytesToCopy;
@@ -95,13 +94,13 @@ static constexpr uint64_t min_length[] = {
 
 static const int kMinLengthSize = ABSL_ARRAYSIZE(min_length);
 
-static inline bool cord_ring_enabled() {
-  return cord_internal::cord_ring_buffer_enabled.load(
+static inline bool btree_enabled() {
+  return cord_internal::cord_btree_enabled.load(
       std::memory_order_relaxed);
 }
 
 static inline bool IsRootBalanced(CordRep* node) {
-  if (node->tag != CONCAT) {
+  if (!node->IsConcat()) {
     return true;
   } else if (node->concat()->depth() <= 15) {
     return true;
@@ -138,7 +137,7 @@ static inline CordRep* VerifyTree(CordRep* node) {
 
 // Return the depth of a node
 static int Depth(const CordRep* rep) {
-  if (rep->tag == CONCAT) {
+  if (rep->IsConcat()) {
     return rep->concat()->depth();
   } else {
     return 0;
@@ -171,7 +170,7 @@ static CordRep* RawConcat(CordRep* left, CordRep* right) {
   }
 
   CordRepConcat* rep = new CordRepConcat();
-  rep->tag = CONCAT;
+  rep->tag = cord_internal::CONCAT;
   SetConcatChildren(rep, left, right);
 
   return rep;
@@ -206,36 +205,32 @@ static CordRep* MakeBalancedTree(CordRep** reps, size_t n) {
 }
 
 static CordRepFlat* CreateFlat(const char* data, size_t length,
-                            size_t alloc_hint) {
+                               size_t alloc_hint) {
   CordRepFlat* flat = CordRepFlat::New(length + alloc_hint);
   flat->length = length;
   memcpy(flat->Data(), data, length);
   return flat;
 }
 
-// Creates a new flat or ringbuffer out of the specified array.
+// Creates a new flat or Btree out of the specified array.
 // The returned node has a refcount of 1.
-static CordRep* RingNewTree(const char* data, size_t length,
-                            size_t alloc_hint) {
+static CordRep* NewBtree(const char* data, size_t length, size_t alloc_hint) {
   if (length <= kMaxFlatLength) {
     return CreateFlat(data, length, alloc_hint);
   }
   CordRepFlat* flat = CreateFlat(data, kMaxFlatLength, 0);
   data += kMaxFlatLength;
   length -= kMaxFlatLength;
-  size_t extra = (length - 1) / kMaxFlatLength + 1;
-  auto* root = CordRepRing::Create(flat, extra);
-  return CordRepRing::Append(root, {data, length}, alloc_hint);
+  auto* root = CordRepBtree::Create(flat);
+  return CordRepBtree::Append(root, {data, length}, alloc_hint);
 }
 
 // Create a new tree out of the specified array.
 // The returned node has a refcount of 1.
-static CordRep* NewTree(const char* data,
-                        size_t length,
-                        size_t alloc_hint) {
+static CordRep* NewTree(const char* data, size_t length, size_t alloc_hint) {
   if (length == 0) return nullptr;
-  if (cord_ring_enabled()) {
-    return RingNewTree(data, length, alloc_hint);
+  if (btree_enabled()) {
+    return NewBtree(data, length, alloc_hint);
   }
   absl::FixedArray<CordRep*> reps((length - 1) / kMaxFlatLength + 1);
   size_t n = 0;
@@ -272,13 +267,42 @@ static CordRep* NewSubstring(CordRep* child, size_t offset, size_t length) {
     CordRepSubstring* rep = new CordRepSubstring();
     assert((offset + length) <= child->length);
     rep->length = length;
-    rep->tag = SUBSTRING;
+    rep->tag = cord_internal::SUBSTRING;
     rep->start = offset;
     rep->child = child;
     return VerifyTree(rep);
   }
 }
 
+// Creates a CordRep from the provided string. If the string is large enough,
+// and not wasteful, we move the string into an external cord rep, preserving
+// the already allocated string contents.
+// Requires the provided string length to be larger than `kMaxInline`.
+static CordRep* CordRepFromString(std::string&& src) {
+  assert(src.length() > cord_internal::kMaxInline);
+  if (
+      // String is short: copy data to avoid external block overhead.
+      src.size() <= kMaxBytesToCopy ||
+      // String is wasteful: copy data to avoid pinning too much unused memory.
+      src.size() < src.capacity() / 2
+  ) {
+    return NewTree(src.data(), src.size(), 0);
+  }
+
+  struct StringReleaser {
+    void operator()(absl::string_view /* data */) {}
+    std::string data;
+  };
+  const absl::string_view original_data = src;
+  auto* rep =
+      static_cast<::absl::cord_internal::CordRepExternalImpl<StringReleaser>*>(
+          absl::cord_internal::NewExternalRep(original_data,
+                                              StringReleaser{std::move(src)}));
+  // Moving src may have invalidated its data pointer, so adjust it.
+  rep->base = rep->template get<0>().data.data();
+  return rep;
+}
+
 // --------------------------------------------------------------------
 // Cord::InlineRep functions
 
@@ -299,20 +323,6 @@ inline char* Cord::InlineRep::set_data(size_t n) {
   return data_.as_chars();
 }
 
-inline CordRep* Cord::InlineRep::force_tree(size_t extra_hint) {
-  if (data_.is_tree()) {
-    return data_.as_tree();
-  }
-
-  size_t len = inline_size();
-  CordRepFlat* result = CordRepFlat::New(len + extra_hint);
-  result->length = len;
-  static_assert(kMinFlatLength >= sizeof(data_), "");
-  memcpy(result->Data(), data_.as_chars(), sizeof(data_));
-  set_tree(result);
-  return result;
-}
-
 inline void Cord::InlineRep::reduce_size(size_t n) {
   size_t tag = inline_size();
   assert(tag <= kMaxInline);
@@ -328,31 +338,78 @@ inline void Cord::InlineRep::remove_prefix(size_t n) {
   reduce_size(n);
 }
 
-// Returns `rep` converted into a CordRepRing.
-// Directly returns `rep` if `rep` is already a CordRepRing.
-static CordRepRing* ForceRing(CordRep* rep, size_t extra) {
-  return (rep->tag == RING) ? rep->ring() : CordRepRing::Create(rep, extra);
+// Returns `rep` converted into a CordRepBtree.
+// Directly returns `rep` if `rep` is already a CordRepBtree.
+static CordRepBtree* ForceBtree(CordRep* rep) {
+  return rep->IsBtree() ? rep->btree() : CordRepBtree::Create(rep);
 }
 
-void Cord::InlineRep::AppendTree(CordRep* tree) {
+void Cord::InlineRep::AppendTreeToInlined(CordRep* tree,
+                                          MethodIdentifier method) {
+  assert(!is_tree());
+  if (!data_.is_empty()) {
+    CordRepFlat* flat = MakeFlatWithExtraCapacity(0);
+    if (btree_enabled()) {
+      tree = CordRepBtree::Append(CordRepBtree::Create(flat), tree);
+    } else {
+      tree = Concat(flat, tree);
+    }
+  }
+  EmplaceTree(tree, method);
+}
+
+void Cord::InlineRep::AppendTreeToTree(CordRep* tree, MethodIdentifier method) {
+  assert(is_tree());
+  const CordzUpdateScope scope(data_.cordz_info(), method);
+  if (btree_enabled()) {
+    tree = CordRepBtree::Append(ForceBtree(data_.as_tree()), tree);
+  } else {
+    tree = Concat(data_.as_tree(), tree);
+  }
+  SetTree(tree, scope);
+}
+
+void Cord::InlineRep::AppendTree(CordRep* tree, MethodIdentifier method) {
   if (tree == nullptr) return;
-  if (data_.is_empty()) {
-    set_tree(tree);
-  } else if (cord_ring_enabled()) {
-    set_tree(CordRepRing::Append(ForceRing(force_tree(0), 1), tree));
+  if (data_.is_tree()) {
+    AppendTreeToTree(tree, method);
   } else {
-    set_tree(Concat(force_tree(0), tree));
+    AppendTreeToInlined(tree, method);
+  }
+}
+
+void Cord::InlineRep::PrependTreeToInlined(CordRep* tree,
+                                           MethodIdentifier method) {
+  assert(!is_tree());
+  if (!data_.is_empty()) {
+    CordRepFlat* flat = MakeFlatWithExtraCapacity(0);
+    if (btree_enabled()) {
+      tree = CordRepBtree::Prepend(CordRepBtree::Create(flat), tree);
+    } else {
+      tree = Concat(tree, flat);
+    }
   }
+  EmplaceTree(tree, method);
 }
 
-void Cord::InlineRep::PrependTree(CordRep* tree) {
+void Cord::InlineRep::PrependTreeToTree(CordRep* tree,
+                                        MethodIdentifier method) {
+  assert(is_tree());
+  const CordzUpdateScope scope(data_.cordz_info(), method);
+  if (btree_enabled()) {
+    tree = CordRepBtree::Prepend(ForceBtree(data_.as_tree()), tree);
+  } else {
+    tree = Concat(tree, data_.as_tree());
+  }
+  SetTree(tree, scope);
+}
+
+void Cord::InlineRep::PrependTree(CordRep* tree, MethodIdentifier method) {
   assert(tree != nullptr);
-  if (data_.is_empty()) {
-    set_tree(tree);
-  } else if (cord_ring_enabled()) {
-    set_tree(CordRepRing::Prepend(ForceRing(force_tree(0), 1), tree));
+  if (data_.is_tree()) {
+    PrependTreeToTree(tree, method);
   } else {
-    set_tree(Concat(tree, force_tree(0)));
+    PrependTreeToInlined(tree, method);
   }
 }
 
@@ -362,8 +419,8 @@ void Cord::InlineRep::PrependTree(CordRep* tree) {
 // written to region and the actual size increase will be written to size.
 static inline bool PrepareAppendRegion(CordRep* root, char** region,
                                        size_t* size, size_t max_length) {
-  if (root->tag == RING && root->refcount.IsOne()) {
-    Span<char> span = root->ring()->GetAppendBuffer(max_length);
+  if (root->IsBtree() && root->refcount.IsMutable()) {
+    Span<char> span = root->btree()->GetAppendBuffer(max_length);
     if (!span.empty()) {
       *region = span.data();
       *size = span.size();
@@ -373,11 +430,11 @@ static inline bool PrepareAppendRegion(CordRep* root, char** region,
 
   // Search down the right-hand path for a non-full FLAT node.
   CordRep* dst = root;
-  while (dst->tag == CONCAT && dst->refcount.IsOne()) {
+  while (dst->IsConcat() && dst->refcount.IsMutable()) {
     dst = dst->concat()->right;
   }
 
-  if (dst->tag < FLAT || !dst->refcount.IsOne()) {
+  if (!dst->IsFlat() || !dst->refcount.IsMutable()) {
     *region = nullptr;
     *size = 0;
     return false;
@@ -404,148 +461,140 @@ static inline bool PrepareAppendRegion(CordRep* root, char** region,
   return true;
 }
 
+template <bool has_length>
 void Cord::InlineRep::GetAppendRegion(char** region, size_t* size,
-                                      size_t max_length) {
-  if (max_length == 0) {
-    *region = nullptr;
-    *size = 0;
-    return;
-  }
-
-  // Try to fit in the inline buffer if possible.
-  if (!is_tree()) {
-    size_t inline_length = inline_size();
-    if (max_length <= kMaxInline - inline_length) {
-      *region = data_.as_chars() + inline_length;
-      *size = max_length;
-      set_inline_size(inline_length + max_length);
+                                      size_t length) {
+  auto constexpr method = CordzUpdateTracker::kGetAppendRegion;
+
+  CordRep* root = tree();
+  size_t sz = root ? root->length : inline_size();
+  if (root == nullptr) {
+    size_t available = kMaxInline - sz;
+    if (available >= (has_length ? length : 1)) {
+      *region = data_.as_chars() + sz;
+      *size = has_length ? length : available;
+      set_inline_size(has_length ? sz + length : kMaxInline);
       return;
     }
   }
 
-  CordRep* root = force_tree(max_length);
-
-  if (PrepareAppendRegion(root, region, size, max_length)) {
+  size_t extra = has_length ? length : (std::max)(sz, kMinFlatLength);
+  CordRep* rep = root ? root : MakeFlatWithExtraCapacity(extra);
+  CordzUpdateScope scope(root ? data_.cordz_info() : nullptr, method);
+  if (PrepareAppendRegion(rep, region, size, length)) {
+    CommitTree(root, rep, scope, method);
     return;
   }
 
   // Allocate new node.
-  CordRepFlat* new_node =
-      CordRepFlat::New(std::max(static_cast<size_t>(root->length), max_length));
-  new_node->length = std::min(new_node->Capacity(), max_length);
+  CordRepFlat* new_node = CordRepFlat::New(extra);
+  new_node->length = std::min(new_node->Capacity(), length);
   *region = new_node->Data();
   *size = new_node->length;
 
-  if (cord_ring_enabled()) {
-    replace_tree(CordRepRing::Append(ForceRing(root, 1), new_node));
-    return;
+  if (btree_enabled()) {
+    rep = CordRepBtree::Append(ForceBtree(rep), new_node);
+  } else {
+    rep = Concat(rep, new_node);
   }
-  replace_tree(Concat(root, new_node));
+  CommitTree(root, rep, scope, method);
 }
 
-void Cord::InlineRep::GetAppendRegion(char** region, size_t* size) {
-  const size_t max_length = std::numeric_limits<size_t>::max();
-
-  // Try to fit in the inline buffer if possible.
-  if (!data_.is_tree()) {
-    size_t inline_length = inline_size();
-    if (inline_length < kMaxInline) {
-      *region = data_.as_chars() + inline_length;
-      *size = kMaxInline - inline_length;
-      set_inline_size(kMaxInline);
-      return;
-    }
+// Computes the memory side of the provided edge which must be a valid data edge
+// for a btrtee, i.e., a FLAT, EXTERNAL or SUBSTRING of a FLAT or EXTERNAL node.
+static bool RepMemoryUsageDataEdge(const CordRep* rep,
+                                   size_t* total_mem_usage) {
+  size_t maybe_sub_size = 0;
+  if (ABSL_PREDICT_FALSE(rep->IsSubstring())) {
+    maybe_sub_size = sizeof(cord_internal::CordRepSubstring);
+    rep = rep->substring()->child;
   }
-
-  CordRep* root = force_tree(max_length);
-
-  if (PrepareAppendRegion(root, region, size, max_length)) {
-    return;
+  if (rep->IsFlat()) {
+    *total_mem_usage += maybe_sub_size + rep->flat()->AllocatedSize();
+    return true;
   }
-
-  // Allocate new node.
-  CordRepFlat* new_node = CordRepFlat::New(root->length);
-  new_node->length = new_node->Capacity();
-  *region = new_node->Data();
-  *size = new_node->length;
-
-  if (cord_ring_enabled()) {
-    replace_tree(CordRepRing::Append(ForceRing(root, 1), new_node));
-    return;
+  if (rep->IsExternal()) {
+    // We don't know anything about the embedded / bound data, but we can safely
+    // assume it is 'at least' a word / pointer to data. In the future we may
+    // choose to use the 'data' byte as a tag to identify the types of some
+    // well-known externals, such as a std::string instance.
+    *total_mem_usage += maybe_sub_size +
+                        sizeof(cord_internal::CordRepExternalImpl<intptr_t>) +
+                        rep->length;
+    return true;
   }
-  replace_tree(Concat(root, new_node));
+  return false;
 }
 
 // If the rep is a leaf, this will increment the value at total_mem_usage and
 // will return true.
 static bool RepMemoryUsageLeaf(const CordRep* rep, size_t* total_mem_usage) {
-  if (rep->tag >= FLAT) {
+  if (rep->IsFlat()) {
     *total_mem_usage += rep->flat()->AllocatedSize();
     return true;
   }
-  if (rep->tag == EXTERNAL) {
-    *total_mem_usage += sizeof(CordRepConcat) + rep->length;
+  if (rep->IsExternal()) {
+    // We don't know anything about the embedded / bound data, but we can safely
+    // assume it is 'at least' a word / pointer to data. In the future we may
+    // choose to use the 'data' byte as a tag to identify the types of some
+    // well-known externals, such as a std::string instance.
+    *total_mem_usage +=
+        sizeof(cord_internal::CordRepExternalImpl<intptr_t>) + rep->length;
     return true;
   }
   return false;
 }
 
 void Cord::InlineRep::AssignSlow(const Cord::InlineRep& src) {
-  ClearSlow();
+  assert(&src != this);
+  assert(is_tree() || src.is_tree());
+  auto constexpr method = CordzUpdateTracker::kAssignCord;
+  if (ABSL_PREDICT_TRUE(!is_tree())) {
+    EmplaceTree(CordRep::Ref(src.as_tree()), src.data_, method);
+    return;
+  }
 
-  data_ = src.data_;
-  if (is_tree()) {
-    data_.set_profiled(false);
-    CordRep::Ref(tree());
-    clear_cordz_info();
+  CordRep* tree = as_tree();
+  if (CordRep* src_tree = src.tree()) {
+    // Leave any existing `cordz_info` in place, and let MaybeTrackCord()
+    // decide if this cord should be (or remains to be) sampled or not.
+    data_.set_tree(CordRep::Ref(src_tree));
+    CordzInfo::MaybeTrackCord(data_, src.data_, method);
+  } else {
+    CordzInfo::MaybeUntrackCord(data_.cordz_info());
+    data_ = src.data_;
   }
+  CordRep::Unref(tree);
 }
 
-void Cord::InlineRep::ClearSlow() {
+void Cord::InlineRep::UnrefTree() {
   if (is_tree()) {
+    CordzInfo::MaybeUntrackCord(data_.cordz_info());
     CordRep::Unref(tree());
   }
-  ResetToEmpty();
 }
 
 // --------------------------------------------------------------------
 // Constructors and destructors
 
-Cord::Cord(absl::string_view src) {
+Cord::Cord(absl::string_view src, MethodIdentifier method)
+    : contents_(InlineData::kDefaultInit) {
   const size_t n = src.size();
   if (n <= InlineRep::kMaxInline) {
-    contents_.set_data(src.data(), n, false);
+    contents_.set_data(src.data(), n, true);
   } else {
-    contents_.set_tree(NewTree(src.data(), n, 0));
+    CordRep* rep = NewTree(src.data(), n, 0);
+    contents_.EmplaceTree(rep, method);
   }
 }
 
 template <typename T, Cord::EnableIfString<T>>
-Cord::Cord(T&& src) {
-  if (
-      // String is short: copy data to avoid external block overhead.
-      src.size() <= kMaxBytesToCopy ||
-      // String is wasteful: copy data to avoid pinning too much unused memory.
-      src.size() < src.capacity() / 2
-  ) {
-    if (src.size() <= InlineRep::kMaxInline) {
-      contents_.set_data(src.data(), src.size(), false);
-    } else {
-      contents_.set_tree(NewTree(src.data(), src.size(), 0));
-    }
+Cord::Cord(T&& src) : contents_(InlineData::kDefaultInit) {
+  if (src.size() <= InlineRep::kMaxInline) {
+    contents_.set_data(src.data(), src.size(), true);
   } else {
-    struct StringReleaser {
-      void operator()(absl::string_view /* data */) {}
-      std::string data;
-    };
-    const absl::string_view original_data = src;
-    auto* rep = static_cast<
-        ::absl::cord_internal::CordRepExternalImpl<StringReleaser>*>(
-        absl::cord_internal::NewExternalRep(
-            original_data, StringReleaser{std::forward<T>(src)}));
-    // Moving src may have invalidated its data pointer, so adjust it.
-    rep->base = rep->template get<0>().data.data();
-    contents_.set_tree(rep);
+    CordRep* rep = CordRepFromString(std::forward<T>(src));
+    contents_.EmplaceTree(rep, CordzUpdateTracker::kConstructorString);
   }
 }
 
@@ -554,9 +603,9 @@ template Cord::Cord(std::string&& src);
 // The destruction code is separate so that the compiler can determine
 // that it does not need to call the destructor on a moved-from Cord.
 void Cord::DestroyCordSlow() {
-  if (CordRep* tree = contents_.tree()) {
-    CordRep::Unref(VerifyTree(tree));
-  }
+  assert(contents_.is_tree());
+  CordzInfo::MaybeUntrackCord(contents_.cordz_info());
+  CordRep::Unref(VerifyTree(contents_.as_tree()));
 }
 
 // --------------------------------------------------------------------
@@ -568,109 +617,116 @@ void Cord::Clear() {
   }
 }
 
-Cord& Cord::operator=(absl::string_view src) {
+Cord& Cord::AssignLargeString(std::string&& src) {
+  auto constexpr method = CordzUpdateTracker::kAssignString;
+  assert(src.size() > kMaxBytesToCopy);
+  CordRep* rep = CordRepFromString(std::move(src));
+  if (CordRep* tree = contents_.tree()) {
+    CordzUpdateScope scope(contents_.cordz_info(), method);
+    contents_.SetTree(rep, scope);
+    CordRep::Unref(tree);
+  } else {
+    contents_.EmplaceTree(rep, method);
+  }
+  return *this;
+}
 
+Cord& Cord::operator=(absl::string_view src) {
+  auto constexpr method = CordzUpdateTracker::kAssignString;
   const char* data = src.data();
   size_t length = src.size();
   CordRep* tree = contents_.tree();
   if (length <= InlineRep::kMaxInline) {
-    // Embed into this->contents_
+    // Embed into this->contents_, which is somewhat subtle:
+    // - MaybeUntrackCord must be called before Unref(tree).
+    // - MaybeUntrackCord must be called before set_data() clobbers cordz_info.
+    // - set_data() must be called before Unref(tree) as it may reference tree.
+    if (tree != nullptr) CordzInfo::MaybeUntrackCord(contents_.cordz_info());
     contents_.set_data(data, length, true);
-    if (tree) CordRep::Unref(tree);
-    return *this;
-  }
-  if (tree != nullptr && tree->tag >= FLAT &&
-      tree->flat()->Capacity() >= length &&
-      tree->refcount.IsOne()) {
-    // Copy in place if the existing FLAT node is reusable.
-    memmove(tree->flat()->Data(), data, length);
-    tree->length = length;
-    VerifyTree(tree);
+    if (tree != nullptr) CordRep::Unref(tree);
     return *this;
   }
-  contents_.set_tree(NewTree(data, length, 0));
-  if (tree) CordRep::Unref(tree);
-  return *this;
-}
-
-template <typename T, Cord::EnableIfString<T>>
-Cord& Cord::operator=(T&& src) {
-  if (src.size() <= kMaxBytesToCopy) {
-    *this = absl::string_view(src);
+  if (tree != nullptr) {
+    CordzUpdateScope scope(contents_.cordz_info(), method);
+    if (tree->IsFlat() && tree->flat()->Capacity() >= length &&
+        tree->refcount.IsMutable()) {
+      // Copy in place if the existing FLAT node is reusable.
+      memmove(tree->flat()->Data(), data, length);
+      tree->length = length;
+      VerifyTree(tree);
+      return *this;
+    }
+    contents_.SetTree(NewTree(data, length, 0), scope);
+    CordRep::Unref(tree);
   } else {
-    *this = Cord(std::forward<T>(src));
+    contents_.EmplaceTree(NewTree(data, length, 0), method);
   }
   return *this;
 }
 
-template Cord& Cord::operator=(std::string&& src);
-
 // TODO(sanjay): Move to Cord::InlineRep section of file.  For now,
 // we keep it here to make diffs easier.
-void Cord::InlineRep::AppendArray(const char* src_data, size_t src_size) {
-  if (src_size == 0) return;  // memcpy(_, nullptr, 0) is undefined.
+void Cord::InlineRep::AppendArray(absl::string_view src,
+                                  MethodIdentifier method) {
+  if (src.empty()) return;  // memcpy(_, nullptr, 0) is undefined.
 
   size_t appended = 0;
-  CordRep* root = nullptr;
-  if (is_tree()) {
-    root = data_.as_tree();
+  CordRep* rep = tree();
+  const CordRep* const root = rep;
+  CordzUpdateScope scope(root ? cordz_info() : nullptr, method);
+  if (root != nullptr) {
     char* region;
-    if (PrepareAppendRegion(root, &region, &appended, src_size)) {
-      memcpy(region, src_data, appended);
+    if (PrepareAppendRegion(rep, &region, &appended, src.size())) {
+      memcpy(region, src.data(), appended);
     }
   } else {
     // Try to fit in the inline buffer if possible.
     size_t inline_length = inline_size();
-    if (src_size <= kMaxInline - inline_length) {
+    if (src.size() <= kMaxInline - inline_length) {
       // Append new data to embedded array
-      memcpy(data_.as_chars() + inline_length, src_data, src_size);
-      set_inline_size(inline_length + src_size);
+      memcpy(data_.as_chars() + inline_length, src.data(), src.size());
+      set_inline_size(inline_length + src.size());
       return;
     }
 
-    // It is possible that src_data == data_, but when we transition from an
-    // InlineRep to a tree we need to assign data_ = root via set_tree. To
-    // avoid corrupting the source data before we copy it, delay calling
-    // set_tree until after we've copied data.
-    // We are going from an inline size to beyond inline size. Make the new size
-    // either double the inlined size, or the added size + 10%.
-    const size_t size1 = inline_length * 2 + src_size;
-    const size_t size2 = inline_length + src_size / 10;
-    root = CordRepFlat::New(std::max<size_t>(size1, size2));
-    appended = std::min(
-        src_size, root->flat()->Capacity() - inline_length);
-    memcpy(root->flat()->Data(), data_.as_chars(), inline_length);
-    memcpy(root->flat()->Data() + inline_length, src_data, appended);
-    root->length = inline_length + appended;
-    set_tree(root);
-  }
-
-  src_data += appended;
-  src_size -= appended;
-  if (src_size == 0) {
-    return;
+    // Allocate flat to be a perfect fit on first append exceeding inlined size.
+    // Subsequent growth will use amortized growth until we reach maximum flat
+    // size.
+    rep = CordRepFlat::New(inline_length + src.size());
+    appended = std::min(src.size(), rep->flat()->Capacity() - inline_length);
+    memcpy(rep->flat()->Data(), data_.as_chars(), inline_length);
+    memcpy(rep->flat()->Data() + inline_length, src.data(), appended);
+    rep->length = inline_length + appended;
   }
 
-  if (cord_ring_enabled()) {
-    absl::string_view data(src_data, src_size);
-    root = ForceRing(root, (data.size() - 1) / kMaxFlatLength + 1);
-    replace_tree(CordRepRing::Append(root->ring(), data));
+  src.remove_prefix(appended);
+  if (src.empty()) {
+    CommitTree(root, rep, scope, method);
     return;
   }
 
-  // Use new block(s) for any remaining bytes that were not handled above.
-  // Alloc extra memory only if the right child of the root of the new tree is
-  // going to be a FLAT node, which will permit further inplace appends.
-  size_t length = src_size;
-  if (src_size < kMaxFlatLength) {
-    // The new length is either
-    // - old size + 10%
-    // - old_size + src_size
-    // This will cause a reasonable conservative step-up in size that is still
-    // large enough to avoid excessive amounts of small fragments being added.
-    length = std::max<size_t>(root->length / 10, src_size);
+  if (btree_enabled()) {
+    // TODO(b/192061034): keep legacy 10% growth rate: consider other rates.
+    rep = ForceBtree(rep);
+    const size_t min_growth = std::max<size_t>(rep->length / 10, src.size());
+    rep = CordRepBtree::Append(rep->btree(), src, min_growth - src.size());
+  } else {
+    // Use new block(s) for any remaining bytes that were not handled above.
+    // Alloc extra memory only if the right child of the root of the new tree
+    // is going to be a FLAT node, which will permit further inplace appends.
+    size_t length = src.size();
+    if (src.size() < kMaxFlatLength) {
+      // The new length is either
+      // - old size + 10%
+      // - old_size + src.size()
+      // This will cause a reasonable conservative step-up in size that is
+      // still large enough to avoid excessive amounts of small fragments
+      // being added.
+      length = std::max<size_t>(rep->length / 10, src.size());
+    }
+    rep = Concat(rep, NewTree(src.data(), src.size(), length - src.size()));
   }
-  set_tree(Concat(root, NewTree(src_data, src_size, length - src_size)));
+  CommitTree(root, rep, scope, method);
 }
 
 inline CordRep* Cord::TakeRep() const& {
@@ -685,10 +741,17 @@ inline CordRep* Cord::TakeRep() && {
 
 template <typename C>
 inline void Cord::AppendImpl(C&& src) {
+  auto constexpr method = CordzUpdateTracker::kAppendCord;
   if (empty()) {
-    // In case of an empty destination avoid allocating a new node, do not copy
-    // data.
-    *this = std::forward<C>(src);
+    // Since destination is empty, we can avoid allocating a node,
+    if (src.contents_.is_tree()) {
+      // by taking the tree directly
+      CordRep* rep = std::forward<C>(src).TakeRep();
+      contents_.EmplaceTree(rep, method);
+    } else {
+      // or copying over inline data
+      contents_.data_ = src.contents_.data_;
+    }
     return;
   }
 
@@ -698,12 +761,12 @@ inline void Cord::AppendImpl(C&& src) {
     CordRep* src_tree = src.contents_.tree();
     if (src_tree == nullptr) {
       // src has embedded data.
-      contents_.AppendArray(src.contents_.data(), src_size);
+      contents_.AppendArray({src.contents_.data(), src_size}, method);
       return;
     }
-    if (src_tree->tag >= FLAT) {
+    if (src_tree->IsFlat()) {
       // src tree just has one flat node.
-      contents_.AppendArray(src_tree->flat()->Data(), src_size);
+      contents_.AppendArray({src_tree->flat()->Data(), src_size}, method);
       return;
     }
     if (&src == this) {
@@ -719,19 +782,25 @@ inline void Cord::AppendImpl(C&& src) {
   }
 
   // Guaranteed to be a tree (kMaxBytesToCopy > kInlinedSize)
-  contents_.AppendTree(std::forward<C>(src).TakeRep());
+  CordRep* rep = std::forward<C>(src).TakeRep();
+  contents_.AppendTree(rep, CordzUpdateTracker::kAppendCord);
 }
 
-void Cord::Append(const Cord& src) { AppendImpl(src); }
+void Cord::Append(const Cord& src) {
+  AppendImpl(src);
+}
 
-void Cord::Append(Cord&& src) { AppendImpl(std::move(src)); }
+void Cord::Append(Cord&& src) {
+  AppendImpl(std::move(src));
+}
 
 template <typename T, Cord::EnableIfString<T>>
 void Cord::Append(T&& src) {
   if (src.size() <= kMaxBytesToCopy) {
     Append(absl::string_view(src));
   } else {
-    Append(Cord(std::forward<T>(src)));
+    CordRep* rep = CordRepFromString(std::forward<T>(src));
+    contents_.AppendTree(rep, CordzUpdateTracker::kAppendString);
   }
 }
 
@@ -741,7 +810,7 @@ void Cord::Prepend(const Cord& src) {
   CordRep* src_tree = src.contents_.tree();
   if (src_tree != nullptr) {
     CordRep::Ref(src_tree);
-    contents_.PrependTree(src_tree);
+    contents_.PrependTree(src_tree, CordzUpdateTracker::kPrependCord);
     return;
   }
 
@@ -750,7 +819,7 @@ void Cord::Prepend(const Cord& src) {
   return Prepend(src_contents);
 }
 
-void Cord::Prepend(absl::string_view src) {
+void Cord::PrependArray(absl::string_view src, MethodIdentifier method) {
   if (src.empty()) return;  // memcpy(_, nullptr, 0) is undefined.
   if (!contents_.is_tree()) {
     size_t cur_size = contents_.inline_size();
@@ -764,7 +833,8 @@ void Cord::Prepend(absl::string_view src) {
       return;
     }
   }
-  contents_.PrependTree(NewTree(src.data(), src.size(), 0));
+  CordRep* rep = NewTree(src.data(), src.size(), 0);
+  contents_.PrependTree(rep, method);
 }
 
 template <typename T, Cord::EnableIfString<T>>
@@ -772,7 +842,8 @@ inline void Cord::Prepend(T&& src) {
   if (src.size() <= kMaxBytesToCopy) {
     Prepend(absl::string_view(src));
   } else {
-    Prepend(Cord(std::forward<T>(src)));
+    CordRep* rep = CordRepFromString(std::forward<T>(src));
+    contents_.PrependTree(rep, CordzUpdateTracker::kPrependString);
   }
 }
 
@@ -783,7 +854,7 @@ static CordRep* RemovePrefixFrom(CordRep* node, size_t n) {
   if (n == 0) return CordRep::Ref(node);
   absl::InlinedVector<CordRep*, kInlinedVectorSize> rhs_stack;
 
-  while (node->tag == CONCAT) {
+  while (node->IsConcat()) {
     assert(n <= node->length);
     if (n < node->concat()->left->length) {
       // Push right to stack, descend left.
@@ -802,7 +873,7 @@ static CordRep* RemovePrefixFrom(CordRep* node, size_t n) {
   } else {
     size_t start = n;
     size_t len = node->length - n;
-    if (node->tag == SUBSTRING) {
+    if (node->IsSubstring()) {
       // Consider in-place update of node, similar to in RemoveSuffixFrom().
       start += node->substring()->start;
       node = node->substring()->child;
@@ -823,9 +894,9 @@ static CordRep* RemoveSuffixFrom(CordRep* node, size_t n) {
   if (n >= node->length) return nullptr;
   if (n == 0) return CordRep::Ref(node);
   absl::InlinedVector<CordRep*, kInlinedVectorSize> lhs_stack;
-  bool inplace_ok = node->refcount.IsOne();
+  bool inplace_ok = node->refcount.IsMutable();
 
-  while (node->tag == CONCAT) {
+  while (node->IsConcat()) {
     assert(n <= node->length);
     if (n < node->concat()->right->length) {
       // Push left to stack, descend right.
@@ -836,13 +907,13 @@ static CordRep* RemoveSuffixFrom(CordRep* node, size_t n) {
       n -= node->concat()->right->length;
       node = node->concat()->left;
     }
-    inplace_ok = inplace_ok && node->refcount.IsOne();
+    inplace_ok = inplace_ok && node->refcount.IsMutable();
   }
   assert(n <= node->length);
 
   if (n == 0) {
     CordRep::Ref(node);
-  } else if (inplace_ok && node->tag != EXTERNAL) {
+  } else if (inplace_ok && !node->IsExternal()) {
     // Consider making a new buffer if the current node capacity is much
     // larger than the new length.
     CordRep::Ref(node);
@@ -850,7 +921,7 @@ static CordRep* RemoveSuffixFrom(CordRep* node, size_t n) {
   } else {
     size_t start = 0;
     size_t len = node->length - n;
-    if (node->tag == SUBSTRING) {
+    if (node->IsSubstring()) {
       start = node->substring()->start;
       node = node->substring()->child;
     }
@@ -870,12 +941,19 @@ void Cord::RemovePrefix(size_t n) {
   CordRep* tree = contents_.tree();
   if (tree == nullptr) {
     contents_.remove_prefix(n);
-  } else if (tree->tag == RING) {
-    contents_.replace_tree(CordRepRing::RemovePrefix(tree->ring(), n));
   } else {
-    CordRep* newrep = RemovePrefixFrom(tree, n);
-    CordRep::Unref(tree);
-    contents_.replace_tree(VerifyTree(newrep));
+    auto constexpr method = CordzUpdateTracker::kRemovePrefix;
+    CordzUpdateScope scope(contents_.cordz_info(), method);
+    if (tree->IsBtree()) {
+      CordRep* old = tree;
+      tree = tree->btree()->SubTree(n, tree->length - n);
+      CordRep::Unref(old);
+    } else {
+      CordRep* newrep = RemovePrefixFrom(tree, n);
+      CordRep::Unref(tree);
+      tree = VerifyTree(newrep);
+    }
+    contents_.SetTreeOrEmpty(tree, scope);
   }
 }
 
@@ -886,12 +964,17 @@ void Cord::RemoveSuffix(size_t n) {
   CordRep* tree = contents_.tree();
   if (tree == nullptr) {
     contents_.reduce_size(n);
-  } else if (tree->tag == RING) {
-    contents_.replace_tree(CordRepRing::RemoveSuffix(tree->ring(), n));
   } else {
-    CordRep* newrep = RemoveSuffixFrom(tree, n);
-    CordRep::Unref(tree);
-    contents_.replace_tree(VerifyTree(newrep));
+    auto constexpr method = CordzUpdateTracker::kRemoveSuffix;
+    CordzUpdateScope scope(contents_.cordz_info(), method);
+    if (tree->IsBtree()) {
+      tree = CordRepBtree::RemoveSuffix(tree->btree(), n);
+    } else {
+      CordRep* newrep = RemoveSuffixFrom(tree, n);
+      CordRep::Unref(tree);
+      tree = VerifyTree(newrep);
+    }
+    contents_.SetTreeOrEmpty(tree, scope);
   }
 }
 
@@ -924,8 +1007,8 @@ static CordRep* NewSubRange(CordRep* node, size_t pos, size_t n) {
       results.push_back(Concat(left, right));
     } else if (pos == 0 && n == node->length) {
       results.push_back(CordRep::Ref(node));
-    } else if (node->tag != CONCAT) {
-      if (node->tag == SUBSTRING) {
+    } else if (!node->IsConcat()) {
+      if (node->IsSubstring()) {
         pos += node->substring()->start;
         node = node->substring()->child;
       }
@@ -951,17 +1034,20 @@ Cord Cord::Subcord(size_t pos, size_t new_size) const {
   size_t length = size();
   if (pos > length) pos = length;
   if (new_size > length - pos) new_size = length - pos;
+  if (new_size == 0) return sub_cord;
+
   CordRep* tree = contents_.tree();
   if (tree == nullptr) {
     // sub_cord is newly constructed, no need to re-zero-out the tail of
     // contents_ memory.
     sub_cord.contents_.set_data(contents_.data() + pos, new_size, false);
-  } else if (new_size == 0) {
-    // We want to return empty subcord, so nothing to do.
-  } else if (new_size <= InlineRep::kMaxInline) {
+    return sub_cord;
+  }
+
+  if (new_size <= InlineRep::kMaxInline) {
+    char* dest = sub_cord.contents_.data_.as_chars();
     Cord::ChunkIterator it = chunk_begin();
     it.AdvanceBytes(pos);
-    char* dest = sub_cord.contents_.data_.as_chars();
     size_t remaining_size = new_size;
     while (remaining_size > it->size()) {
       cord_internal::SmallMemmove(dest, it->data(), it->size());
@@ -971,12 +1057,16 @@ Cord Cord::Subcord(size_t pos, size_t new_size) const {
     }
     cord_internal::SmallMemmove(dest, it->data(), remaining_size);
     sub_cord.contents_.set_inline_size(new_size);
-  } else if (tree->tag == RING) {
-    tree = CordRepRing::SubRing(CordRep::Ref(tree)->ring(), pos, new_size);
-    sub_cord.contents_.set_tree(tree);
+    return sub_cord;
+  }
+
+  if (tree->IsBtree()) {
+    tree = tree->btree()->SubTree(pos, new_size);
   } else {
-    sub_cord.contents_.set_tree(NewSubRange(tree, pos, new_size));
+    tree = NewSubRange(tree, pos, new_size);
   }
+  sub_cord.contents_.EmplaceTree(tree, contents_.data_,
+                                 CordzUpdateTracker::kSubCord);
   return sub_cord;
 }
 
@@ -995,7 +1085,7 @@ class CordForest {
       CordRep* node = pending.back();
       pending.pop_back();
       CheckNode(node);
-      if (ABSL_PREDICT_FALSE(node->tag != CONCAT)) {
+      if (ABSL_PREDICT_FALSE(!node->IsConcat())) {
         AddNode(node);
         continue;
       }
@@ -1089,7 +1179,7 @@ class CordForest {
 
   static void CheckNode(CordRep* node) {
     ABSL_INTERNAL_CHECK(node->length != 0u, "");
-    if (node->tag == CONCAT) {
+    if (node->IsConcat()) {
       ABSL_INTERNAL_CHECK(node->concat()->left != nullptr, "");
       ABSL_INTERNAL_CHECK(node->concat()->right != nullptr, "");
       ABSL_INTERNAL_CHECK(node->length == (node->concat()->left->length +
@@ -1109,7 +1199,7 @@ class CordForest {
 
 static CordRep* Rebalance(CordRep* node) {
   VerifyTree(node);
-  assert(node->tag == CONCAT);
+  assert(node->IsConcat());
 
   if (node->length == 0) {
     return nullptr;
@@ -1159,28 +1249,33 @@ bool ComputeCompareResult<bool>(int memcmp_res) {
 
 }  // namespace
 
-// Helper routine. Locates the first flat chunk of the Cord without
-// initializing the iterator.
+// Helper routine. Locates the first flat or external chunk of the Cord without
+// initializing the iterator, and returns a string_view referencing the data.
 inline absl::string_view Cord::InlineRep::FindFlatStartPiece() const {
   if (!is_tree()) {
     return absl::string_view(data_.as_chars(), data_.inline_size());
   }
 
   CordRep* node = tree();
-  if (node->tag >= FLAT) {
+  if (node->IsFlat()) {
     return absl::string_view(node->flat()->Data(), node->length);
   }
 
-  if (node->tag == EXTERNAL) {
+  if (node->IsExternal()) {
     return absl::string_view(node->external()->base, node->length);
   }
 
-  if (node->tag == RING) {
-    return node->ring()->entry_data(node->ring()->head());
+  if (node->IsBtree()) {
+    CordRepBtree* tree = node->btree();
+    int height = tree->height();
+    while (--height >= 0) {
+      tree = tree->Edge(CordRepBtree::kFront)->btree();
+    }
+    return tree->Data(tree->begin());
   }
 
   // Walk down the left branches until we hit a non-CONCAT node.
-  while (node->tag == CONCAT) {
+  while (node->IsConcat()) {
     node = node->concat()->left;
   }
 
@@ -1189,16 +1284,16 @@ inline absl::string_view Cord::InlineRep::FindFlatStartPiece() const {
   size_t length = node->length;
   assert(length != 0);
 
-  if (node->tag == SUBSTRING) {
+  if (node->IsSubstring()) {
     offset = node->substring()->start;
     node = node->substring()->child;
   }
 
-  if (node->tag >= FLAT) {
+  if (node->IsFlat()) {
     return absl::string_view(node->flat()->Data() + offset, length);
   }
 
-  assert((node->tag == EXTERNAL) && "Expect FLAT or EXTERNAL node here");
+  assert(node->IsExternal() && "Expect FLAT or EXTERNAL node here");
 
   return absl::string_view(node->external()->base + offset, length);
 }
@@ -1392,7 +1487,7 @@ Cord::ChunkIterator& Cord::ChunkIterator::AdvanceStack() {
 
   // Walk down the left branches until we hit a non-CONCAT node. Save the
   // right children to the stack for subsequent traversal.
-  while (node->tag == CONCAT) {
+  while (node->IsConcat()) {
     stack_of_right_children.push_back(node->concat()->right);
     node = node->concat()->left;
   }
@@ -1400,15 +1495,15 @@ Cord::ChunkIterator& Cord::ChunkIterator::AdvanceStack() {
   // Get the child node if we encounter a SUBSTRING.
   size_t offset = 0;
   size_t length = node->length;
-  if (node->tag == SUBSTRING) {
+  if (node->IsSubstring()) {
     offset = node->substring()->start;
     node = node->substring()->child;
   }
 
-  assert(node->tag == EXTERNAL || node->tag >= FLAT);
+  assert(node->IsExternal() || node->IsFlat());
   assert(length != 0);
   const char* data =
-      node->tag == EXTERNAL ? node->external()->base : node->flat()->Data();
+      node->IsExternal() ? node->external()->base : node->flat()->Data();
   current_chunk_ = absl::string_view(data + offset, length);
   current_leaf_ = node;
   return *this;
@@ -1418,6 +1513,7 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
   ABSL_HARDENING_ASSERT(bytes_remaining_ >= n &&
                         "Attempted to iterate past `end()`");
   Cord subcord;
+  auto constexpr method = CordzUpdateTracker::kCordReader;
 
   if (n <= InlineRep::kMaxInline) {
     // Range to read fits in inline data. Flatten it.
@@ -1437,21 +1533,21 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
     return subcord;
   }
 
-  if (ring_reader_) {
+  if (btree_reader_) {
     size_t chunk_size = current_chunk_.size();
     if (n <= chunk_size && n <= kMaxBytesToCopy) {
-      subcord = Cord(current_chunk_.substr(0, n));
-    } else {
-      auto* ring = CordRep::Ref(ring_reader_.ring())->ring();
-      size_t offset = ring_reader_.length() - bytes_remaining_;
-      subcord.contents_.set_tree(CordRepRing::SubRing(ring, offset, n));
-    }
-    if (n < chunk_size) {
-      bytes_remaining_ -= n;
-      current_chunk_.remove_prefix(n);
+      subcord = Cord(current_chunk_.substr(0, n), method);
+      if (n < chunk_size) {
+        current_chunk_.remove_prefix(n);
+      } else {
+        current_chunk_ = btree_reader_.Next();
+      }
     } else {
-      AdvanceBytesRing(n);
+      CordRep* rep;
+      current_chunk_ = btree_reader_.Read(n, chunk_size, rep);
+      subcord.contents_.EmplaceTree(rep, method);
     }
+    bytes_remaining_ -= n;
     return subcord;
   }
 
@@ -1460,10 +1556,10 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
     // Range to read is a proper subrange of the current chunk.
     assert(current_leaf_ != nullptr);
     CordRep* subnode = CordRep::Ref(current_leaf_);
-    const char* data = subnode->tag == EXTERNAL ? subnode->external()->base
-                                                : subnode->flat()->Data();
+    const char* data = subnode->IsExternal() ? subnode->external()->base
+                                             : subnode->flat()->Data();
     subnode = NewSubstring(subnode, current_chunk_.data() - data, n);
-    subcord.contents_.set_tree(VerifyTree(subnode));
+    subcord.contents_.EmplaceTree(VerifyTree(subnode), method);
     RemoveChunkPrefix(n);
     return subcord;
   }
@@ -1473,8 +1569,8 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
   assert(current_leaf_ != nullptr);
   CordRep* subnode = CordRep::Ref(current_leaf_);
   if (current_chunk_.size() < subnode->length) {
-    const char* data = subnode->tag == EXTERNAL ? subnode->external()->base
-                                                : subnode->flat()->Data();
+    const char* data = subnode->IsExternal() ? subnode->external()->base
+                                             : subnode->flat()->Data();
     subnode = NewSubstring(subnode, current_chunk_.data() - data,
                            current_chunk_.size());
   }
@@ -1506,13 +1602,13 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
   if (node == nullptr) {
     // We have reached the end of the Cord.
     assert(bytes_remaining_ == 0);
-    subcord.contents_.set_tree(VerifyTree(subnode));
+    subcord.contents_.EmplaceTree(VerifyTree(subnode), method);
     return subcord;
   }
 
   // Walk down the appropriate branches until we hit a non-CONCAT node. Save the
   // right children to the stack for subsequent traversal.
-  while (node->tag == CONCAT) {
+  while (node->IsConcat()) {
     if (node->concat()->left->length > n) {
       // Push right, descend left.
       stack_of_right_children.push_back(node->concat()->right);
@@ -1529,24 +1625,24 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
   // Get the child node if we encounter a SUBSTRING.
   size_t offset = 0;
   size_t length = node->length;
-  if (node->tag == SUBSTRING) {
+  if (node->IsSubstring()) {
     offset = node->substring()->start;
     node = node->substring()->child;
   }
 
   // Range to read ends with a proper (possibly empty) subrange of the current
   // chunk.
-  assert(node->tag == EXTERNAL || node->tag >= FLAT);
+  assert(node->IsExternal() || node->IsFlat());
   assert(length > n);
   if (n > 0) {
     subnode = Concat(subnode, NewSubstring(CordRep::Ref(node), offset, n));
   }
   const char* data =
-      node->tag == EXTERNAL ? node->external()->base : node->flat()->Data();
+      node->IsExternal() ? node->external()->base : node->flat()->Data();
   current_chunk_ = absl::string_view(data + offset + n, length - n);
   current_leaf_ = node;
   bytes_remaining_ -= n;
-  subcord.contents_.set_tree(VerifyTree(subnode));
+  subcord.contents_.EmplaceTree(VerifyTree(subnode), method);
   return subcord;
 }
 
@@ -1585,7 +1681,7 @@ void Cord::ChunkIterator::AdvanceBytesSlowPath(size_t n) {
 
   // Walk down the appropriate branches until we hit a non-CONCAT node. Save the
   // right children to the stack for subsequent traversal.
-  while (node->tag == CONCAT) {
+  while (node->IsConcat()) {
     if (node->concat()->left->length > n) {
       // Push right, descend left.
       stack_of_right_children.push_back(node->concat()->right);
@@ -1601,15 +1697,15 @@ void Cord::ChunkIterator::AdvanceBytesSlowPath(size_t n) {
   // Get the child node if we encounter a SUBSTRING.
   size_t offset = 0;
   size_t length = node->length;
-  if (node->tag == SUBSTRING) {
+  if (node->IsSubstring()) {
     offset = node->substring()->start;
     node = node->substring()->child;
   }
 
-  assert(node->tag == EXTERNAL || node->tag >= FLAT);
+  assert(node->IsExternal() || node->IsFlat());
   assert(length > n);
   const char* data =
-      node->tag == EXTERNAL ? node->external()->base : node->flat()->Data();
+      node->IsExternal() ? node->external()->base : node->flat()->Data();
   current_chunk_ = absl::string_view(data + offset + n, length - n);
   current_leaf_ = node;
   bytes_remaining_ -= n;
@@ -1625,15 +1721,15 @@ char Cord::operator[](size_t i) const {
   while (true) {
     assert(rep != nullptr);
     assert(offset < rep->length);
-    if (rep->tag >= FLAT) {
+    if (rep->IsFlat()) {
       // Get the "i"th character directly from the flat array.
       return rep->flat()->Data()[offset];
-    } else if (rep->tag == RING) {
-      return rep->ring()->GetCharacter(offset);
-    } else if (rep->tag == EXTERNAL) {
+    } else if (rep->IsBtree()) {
+      return rep->btree()->GetCharacter(offset);
+    } else if (rep->IsExternal()) {
       // Get the "i"th character from the external array.
       return rep->external()->base[offset];
-    } else if (rep->tag == CONCAT) {
+    } else if (rep->IsConcat()) {
       // Recursively branch to the side of the concatenation that the "i"th
       // character is on.
       size_t left_length = rep->concat()->left->length;
@@ -1645,7 +1741,7 @@ char Cord::operator[](size_t i) const {
       }
     } else {
       // This must be a substring a node, so bypass it to get to the child.
-      assert(rep->tag == SUBSTRING);
+      assert(rep->IsSubstring());
       offset += rep->substring()->start;
       rep = rep->substring()->child;
     }
@@ -1653,6 +1749,7 @@ char Cord::operator[](size_t i) const {
 }
 
 absl::string_view Cord::FlattenSlowPath() {
+  assert(contents_.is_tree());
   size_t total_size = size();
   CordRep* new_rep;
   char* new_buffer;
@@ -1673,31 +1770,35 @@ absl::string_view Cord::FlattenSlowPath() {
                                             s.size());
         });
   }
-  if (CordRep* tree = contents_.tree()) {
-    CordRep::Unref(tree);
-  }
-  contents_.set_tree(new_rep);
+  CordzUpdateScope scope(contents_.cordz_info(), CordzUpdateTracker::kFlatten);
+  CordRep::Unref(contents_.as_tree());
+  contents_.SetTree(new_rep, scope);
   return absl::string_view(new_buffer, total_size);
 }
 
 /* static */ bool Cord::GetFlatAux(CordRep* rep, absl::string_view* fragment) {
   assert(rep != nullptr);
-  if (rep->tag >= FLAT) {
+  if (rep->IsFlat()) {
     *fragment = absl::string_view(rep->flat()->Data(), rep->length);
     return true;
-  } else if (rep->tag == EXTERNAL) {
+  } else if (rep->IsExternal()) {
     *fragment = absl::string_view(rep->external()->base, rep->length);
     return true;
-  } else if (rep->tag == SUBSTRING) {
+  } else if (rep->IsBtree()) {
+    return rep->btree()->IsFlat(fragment);
+  } else if (rep->IsSubstring()) {
     CordRep* child = rep->substring()->child;
-    if (child->tag >= FLAT) {
+    if (child->IsFlat()) {
       *fragment = absl::string_view(
           child->flat()->Data() + rep->substring()->start, rep->length);
       return true;
-    } else if (child->tag == EXTERNAL) {
+    } else if (child->IsExternal()) {
       *fragment = absl::string_view(
           child->external()->base + rep->substring()->start, rep->length);
       return true;
+    } else if (child->IsBtree()) {
+      return child->btree()->IsFlat(rep->substring()->start, rep->length,
+                                    fragment);
     }
   }
   return false;
@@ -1706,7 +1807,7 @@ absl::string_view Cord::FlattenSlowPath() {
 /* static */ void Cord::ForEachChunkAux(
     absl::cord_internal::CordRep* rep,
     absl::FunctionRef<void(absl::string_view)> callback) {
-  if (rep->tag == RING) {
+  if (rep->IsBtree()) {
     ChunkIterator it(rep), end;
     while (it != end) {
       callback(*it);
@@ -1722,7 +1823,7 @@ absl::string_view Cord::FlattenSlowPath() {
   absl::cord_internal::CordRep* stack[stack_max];
   absl::cord_internal::CordRep* current_node = rep;
   while (true) {
-    if (current_node->tag == CONCAT) {
+    if (current_node->IsConcat()) {
       if (stack_pos == stack_max) {
         // There's no more room on our stack array to add another right branch,
         // and the idea is to avoid allocations, so call this function
@@ -1769,38 +1870,29 @@ static void DumpNode(CordRep* rep, bool include_data, std::ostream* os,
     *os << "]";
     *os << " " << (IsRootBalanced(rep) ? 'b' : 'u');
     *os << " " << std::setw(indent) << "";
-    if (rep->tag == CONCAT) {
+    if (rep->IsConcat()) {
       *os << "CONCAT depth=" << Depth(rep) << "\n";
       indent += kIndentStep;
       indents.push_back(indent);
       stack.push_back(rep->concat()->right);
       rep = rep->concat()->left;
-    } else if (rep->tag == SUBSTRING) {
+    } else if (rep->IsSubstring()) {
       *os << "SUBSTRING @ " << rep->substring()->start << "\n";
       indent += kIndentStep;
       rep = rep->substring()->child;
     } else {  // Leaf or ring
-      if (rep->tag == EXTERNAL) {
+      if (rep->IsExternal()) {
         *os << "EXTERNAL [";
         if (include_data)
           *os << absl::CEscape(std::string(rep->external()->base, rep->length));
         *os << "]\n";
-      } else if (rep->tag >= FLAT) {
-        *os << "FLAT cap=" << rep->flat()->Capacity()
-            << " [";
+      } else if (rep->IsFlat()) {
+        *os << "FLAT cap=" << rep->flat()->Capacity() << " [";
         if (include_data)
           *os << absl::CEscape(std::string(rep->flat()->Data(), rep->length));
         *os << "]\n";
       } else {
-        assert(rep->tag == RING);
-        auto* ring = rep->ring();
-        *os << "RING, entries = " << ring->entries() << "\n";
-        CordRepRing::index_type head = ring->head();
-        do {
-          DumpNode(ring->entry_child(head), include_data, os,
-                   indent + kIndentStep);
-          head = ring->advance(head);;
-        } while (head != ring->tail());
+        CordRepBtree::Dump(rep, /*label=*/ "", include_data, *os);
       }
       if (stack.empty()) break;
       rep = stack.back();
@@ -1832,7 +1924,7 @@ static bool VerifyNode(CordRep* root, CordRep* start_node,
       ABSL_INTERNAL_CHECK(node->length != 0, ReportError(root, node));
     }
 
-    if (node->tag == CONCAT) {
+    if (node->IsConcat()) {
       ABSL_INTERNAL_CHECK(node->concat()->left != nullptr,
                           ReportError(root, node));
       ABSL_INTERNAL_CHECK(node->concat()->right != nullptr,
@@ -1844,14 +1936,13 @@ static bool VerifyNode(CordRep* root, CordRep* start_node,
         worklist.push_back(node->concat()->right);
         worklist.push_back(node->concat()->left);
       }
-    } else if (node->tag >= FLAT) {
-      ABSL_INTERNAL_CHECK(
-          node->length <= node->flat()->Capacity(),
-          ReportError(root, node));
-    } else if (node->tag == EXTERNAL) {
+    } else if (node->IsFlat()) {
+      ABSL_INTERNAL_CHECK(node->length <= node->flat()->Capacity(),
+                          ReportError(root, node));
+    } else if (node->IsExternal()) {
       ABSL_INTERNAL_CHECK(node->external()->base != nullptr,
                           ReportError(root, node));
-    } else if (node->tag == SUBSTRING) {
+    } else if (node->IsSubstring()) {
       ABSL_INTERNAL_CHECK(
           node->substring()->start < node->substring()->child->length,
           ReportError(root, node));
@@ -1880,7 +1971,7 @@ static bool VerifyNode(CordRep* root, CordRep* start_node,
   while (true) {
     const CordRep* next_node = nullptr;
 
-    if (cur_node->tag == CONCAT) {
+    if (cur_node->IsConcat()) {
       total_mem_usage += sizeof(CordRepConcat);
       const CordRep* left = cur_node->concat()->left;
       if (!RepMemoryUsageLeaf(left, &total_mem_usage)) {
@@ -1894,18 +1985,21 @@ static bool VerifyNode(CordRep* root, CordRep* start_node,
         }
         next_node = right;
       }
-    } else if (cur_node->tag == RING) {
-      total_mem_usage += CordRepRing::AllocSize(cur_node->ring()->capacity());
-      const CordRepRing* ring = cur_node->ring();
-      CordRepRing::index_type pos = ring->head(), tail = ring->tail();
-      do {
-        CordRep* node = ring->entry_child(pos);
-        assert(node->tag >= FLAT || node->tag == EXTERNAL);
-        RepMemoryUsageLeaf(node, &total_mem_usage);
-      } while ((pos = ring->advance(pos)) != tail);
+    } else if (cur_node->IsBtree()) {
+      total_mem_usage += sizeof(CordRepBtree);
+      const CordRepBtree* node = cur_node->btree();
+      if (node->height() == 0) {
+        for (const CordRep* edge : node->Edges()) {
+          RepMemoryUsageDataEdge(edge, &total_mem_usage);
+        }
+      } else {
+        for (const CordRep* edge : node->Edges()) {
+          tree_stack.push_back(edge);
+        }
+      }
     } else {
       // Since cur_node is not a leaf or a concat node it must be a substring.
-      assert(cur_node->tag == SUBSTRING);
+      assert(cur_node->IsSubstring());
       total_mem_usage += sizeof(CordRepSubstring);
       next_node = cur_node->substring()->child;
       if (RepMemoryUsageLeaf(next_node, &total_mem_usage)) {