grpc 1.47.0 → 1.53.0

data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.cc

@@ -17,11 +17,14 @@
 #include <cassert>
 #include <cstdint>
 #include <iostream>
+#include <ostream>
 #include <string>
 
 #include "absl/base/attributes.h"
 #include "absl/base/config.h"
 #include "absl/base/internal/raw_logging.h"
+#include "absl/base/optimization.h"
+#include "absl/strings/internal/cord_data_edge.h"
 #include "absl/strings/internal/cord_internal.h"
 #include "absl/strings/internal/cord_rep_consume.h"
 #include "absl/strings/internal/cord_rep_flat.h"
@@ -32,7 +35,9 @@ namespace absl {
 ABSL_NAMESPACE_BEGIN
 namespace cord_internal {
 
-constexpr size_t CordRepBtree::kMaxCapacity;  // NOLINT: needed for c++ < c++17
+#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL
+constexpr size_t CordRepBtree::kMaxCapacity;
+#endif
 
 namespace {
 
@@ -52,8 +57,10 @@ inline bool exhaustive_validation() {
 // Prints the entire tree structure or 'rep'. External callers should
 // not specify 'depth' and leave it to its default (0) value.
 // Rep may be a CordRepBtree tree, or a SUBSTRING / EXTERNAL / FLAT node.
-void DumpAll(const CordRep* rep, bool include_contents, std::ostream& stream,
-             int depth = 0) {
+void DumpAll(const CordRep* rep,
+             bool include_contents,
+             std::ostream& stream,
+             size_t depth = 0) {
   // Allow for full height trees + substring -> flat / external nodes.
   assert(depth <= CordRepBtree::kMaxDepth + 2);
   std::string sharing = const_cast<CordRep*>(rep)->refcount.IsOne()
@@ -69,7 +76,7 @@ void DumpAll(const CordRep* rep, bool include_contents, std::ostream& stream,
       // indentation and prefix / labels keeps us within roughly 80-100 wide.
       constexpr size_t kMaxDataLength = 60;
       stream << ", data = \""
-             << CordRepBtree::EdgeData(r).substr(0, kMaxDataLength)
+             << EdgeData(r).substr(0, kMaxDataLength)
              << (r->length > kMaxDataLength ? "\"..." : "\"");
     }
     stream << '\n';
@@ -119,6 +126,7 @@ CordRepSubstring* CreateSubstring(CordRep* rep, size_t offset, size_t n) {
     rep = CordRep::Ref(substring->child);
     CordRep::Unref(substring);
   }
+  assert(rep->IsExternal() || rep->IsFlat());
   CordRepSubstring* substring = new CordRepSubstring();
   substring->length = n;
   substring->tag = SUBSTRING;
@@ -149,7 +157,7 @@ inline CordRep* MakeSubstring(CordRep* rep, size_t offset) {
 CordRep* ResizeEdge(CordRep* edge, size_t length, bool is_mutable) {
   assert(length > 0);
   assert(length <= edge->length);
-  assert(CordRepBtree::IsDataEdge(edge));
+  assert(IsDataEdge(edge));
   if (length >= edge->length) return edge;
 
   if (is_mutable && (edge->tag >= FLAT || edge->tag == SUBSTRING)) {
@@ -190,24 +198,29 @@ inline void FastUnref(R* r, Fn&& fn) {
   }
 }
 
-// Deletes a leaf node data edge. Requires `rep` to be an EXTERNAL or FLAT
-// node, or a SUBSTRING of an EXTERNAL or FLAT node.
-void DeleteLeafEdge(CordRep* rep) {
-  for (;;) {
+
+void DeleteSubstring(CordRepSubstring* substring) {
+  CordRep* rep = substring->child;
+  if (!rep->refcount.Decrement()) {
     if (rep->tag >= FLAT) {
       CordRepFlat::Delete(rep->flat());
-      return;
-    }
-    if (rep->tag == EXTERNAL) {
+    } else {
+      assert(rep->tag == EXTERNAL);
       CordRepExternal::Delete(rep->external());
-      return;
     }
-    assert(rep->tag == SUBSTRING);
-    CordRepSubstring* substring = rep->substring();
-    rep = substring->child;
-    assert(rep->tag == EXTERNAL || rep->tag >= FLAT);
-    delete substring;
-    if (rep->refcount.Decrement()) return;
+  }
+  delete substring;
+}
+
+// Deletes a leaf node data edge. Requires `IsDataEdge(rep)`.
+void DeleteLeafEdge(CordRep* rep) {
+  assert(IsDataEdge(rep));
+  if (rep->tag >= FLAT) {
+    CordRepFlat::Delete(rep->flat());
+  } else if (rep->tag == EXTERNAL) {
+    CordRepExternal::Delete(rep->external());
+  } else {
+    DeleteSubstring(rep->substring());
   }
 }
 
@@ -216,8 +229,8 @@ void DeleteLeafEdge(CordRep* rep) {
 // propagate node changes up the stack.
 template <EdgeType edge_type>
 struct StackOperations {
-  // Returns true if the node at 'depth' is mutable, i.e. has a refcount
-  // of one, carries no CRC, and all of its parent nodes have a refcount of one.
+  // Returns true if the node at 'depth' is not shared, i.e. has a refcount
+  // of one and all of its parent nodes have a refcount of one.
   inline bool owned(int depth) const { return depth < share_depth; }
 
   // Returns the node at 'depth'.
@@ -228,11 +241,11 @@ struct StackOperations {
   inline CordRepBtree* BuildStack(CordRepBtree* tree, int depth) {
     assert(depth <= tree->height());
     int current_depth = 0;
-    while (current_depth < depth && tree->refcount.IsMutable()) {
+    while (current_depth < depth && tree->refcount.IsOne()) {
       stack[current_depth++] = tree;
       tree = tree->Edge(edge_type)->btree();
     }
-    share_depth = current_depth + (tree->refcount.IsMutable() ? 1 : 0);
+    share_depth = current_depth + (tree->refcount.IsOne() ? 1 : 0);
     while (current_depth < depth) {
       stack[current_depth++] = tree;
       tree = tree->Edge(edge_type)->btree();
@@ -241,17 +254,17 @@ struct StackOperations {
   }
 
   // Builds a stack with the invariant that all nodes are private owned / not
-  // shared and carry no CRC data. This is used in iterative updates where a
-  // previous propagation guaranteed all nodes have this property.
+  // shared. This is used in iterative updates where a previous propagation
+  // guaranteed all nodes are owned / private.
   inline void BuildOwnedStack(CordRepBtree* tree, int height) {
     assert(height <= CordRepBtree::kMaxHeight);
     int depth = 0;
     while (depth < height) {
-      assert(tree->refcount.IsMutable());
+      assert(tree->refcount.IsOne());
       stack[depth++] = tree;
       tree = tree->Edge(edge_type)->btree();
     }
-    assert(tree->refcount.IsMutable());
+    assert(tree->refcount.IsOne());
     share_depth = depth + 1;
   }
 
@@ -274,7 +287,7 @@ struct StackOperations {
       case CordRepBtree::kSelf:
         return result.tree;
     }
-    ABSL_INTERNAL_UNREACHABLE;
+    ABSL_UNREACHABLE();
     return result.tree;
   }
 
@@ -336,12 +349,12 @@ struct StackOperations {
     return Unwind</*propagate=*/true>(tree, depth, length, result);
   }
 
-  // `share_depth` contains the depth at which the nodes in the stack cannot
-  // be mutated. I.e., if the top most level is shared (i.e.:
-  // `!refcount.IsMutable()`), then `share_depth` is 0. If the 2nd node
-  // is shared (and implicitly all nodes below that) then `share_depth` is 1,
-  // etc. A `share_depth` greater than the depth of the stack indicates that
-  // none of the nodes in the stack are shared.
+  // `share_depth` contains the depth at which the nodes in the stack become
+  // shared. I.e., if the top most level is shared (i.e.: `!refcount.IsOne()`),
+  // then `share_depth` is 0. If the 2nd node is shared (and implicitly all
+  // nodes below that) then `share_depth` is 1, etc. A `share_depth` greater
+  // than the depth of the stack indicates that none of the nodes in the stack
+  // are shared.
   int share_depth;
 
   NodeStack stack;
@@ -372,19 +385,37 @@ void CordRepBtree::Dump(const CordRep* rep, std::ostream& stream) {
   Dump(rep, absl::string_view(), false, stream);
 }
 
-void CordRepBtree::DestroyLeaf(CordRepBtree* tree, size_t begin, size_t end) {
-  for (CordRep* edge : tree->Edges(begin, end)) {
-    FastUnref(edge, DeleteLeafEdge);
+template <size_t size>
+static void DestroyTree(CordRepBtree* tree) {
+  for (CordRep* node : tree->Edges()) {
+    if (node->refcount.Decrement()) continue;
+    for (CordRep* edge : node->btree()->Edges()) {
+      if (edge->refcount.Decrement()) continue;
+      if (size == 1) {
+        DeleteLeafEdge(edge);
+      } else {
+        CordRepBtree::Destroy(edge->btree());
+      }
+    }
+    CordRepBtree::Delete(node->btree());
   }
-  Delete(tree);
+  CordRepBtree::Delete(tree);
 }
 
-void CordRepBtree::DestroyNonLeaf(CordRepBtree* tree, size_t begin,
-                                  size_t end) {
-  for (CordRep* edge : tree->Edges(begin, end)) {
-    FastUnref(edge->btree(), Destroy);
+void CordRepBtree::Destroy(CordRepBtree* tree) {
+  switch (tree->height()) {
+    case 0:
+      for (CordRep* edge : tree->Edges()) {
+        if (!edge->refcount.Decrement()) {
+          DeleteLeafEdge(edge);
+        }
+      }
+      return CordRepBtree::Delete(tree);
+    case 1:
+      return DestroyTree<1>(tree);
+    default:
+      return DestroyTree<2>(tree);
   }
-  Delete(tree);
 }
 
 bool CordRepBtree::IsValid(const CordRepBtree* tree, bool shallow) {
@@ -472,7 +503,7 @@ OpResult CordRepBtree::SetEdge(bool owned, CordRep* edge, size_t delta) {
     // open interval [begin, back) or [begin + 1, end) depending on `edge_type`.
     // We conveniently cover both case using a constexpr `shift` being 0 or 1
     // as `end :== back + 1`.
-    result = {CopyRaw(), kCopied};
+    result = {CopyRaw(length), kCopied};
     constexpr int shift = edge_type == kFront ? 1 : 0;
     for (CordRep* r : Edges(begin() + shift, back() + shift)) {
       CordRep::Ref(r);
@@ -773,7 +804,7 @@ CopyResult CordRepBtree::CopyPrefix(size_t n, bool allow_folding) {
 
 CordRep* CordRepBtree::ExtractFront(CordRepBtree* tree) {
   CordRep* front = tree->Edge(tree->begin());
-  if (tree->refcount.IsMutable()) {
+  if (tree->refcount.IsOne()) {
     Unref(tree->Edges(tree->begin() + 1, tree->end()));
     CordRepBtree::Delete(tree);
   } else {
@@ -786,7 +817,7 @@ CordRep* CordRepBtree::ExtractFront(CordRepBtree* tree) {
 CordRepBtree* CordRepBtree::ConsumeBeginTo(CordRepBtree* tree, size_t end,
                                            size_t new_length) {
   assert(end <= tree->end());
-  if (tree->refcount.IsMutable()) {
+  if (tree->refcount.IsOne()) {
     Unref(tree->Edges(end, tree->end()));
     tree->set_end(end);
     tree->length = new_length;
@@ -813,13 +844,13 @@ CordRep* CordRepBtree::RemoveSuffix(CordRepBtree* tree, size_t n) {
 
   size_t length = len - n;
   int height = tree->height();
-  bool is_mutable = tree->refcount.IsMutable();
+  bool is_mutable = tree->refcount.IsOne();
 
   // Extract all top nodes which are reduced to size = 1
   Position pos = tree->IndexOfLength(length);
   while (pos.index == tree->begin()) {
     CordRep* edge = ExtractFront(tree);
-    is_mutable &= edge->refcount.IsMutable();
+    is_mutable &= edge->refcount.IsOne();
     if (height-- == 0) return ResizeEdge(edge, length, is_mutable);
     tree = edge->btree();
     pos = tree->IndexOfLength(length);
@@ -835,8 +866,8 @@ CordRep* CordRepBtree::RemoveSuffix(CordRepBtree* tree, size_t n) {
   length = pos.n;
   while (length != edge->length) {
     // ConsumeBeginTo guarantees `tree` is a clean, privately owned copy.
-    assert(tree->refcount.IsMutable());
-    const bool edge_is_mutable = edge->refcount.IsMutable();
+    assert(tree->refcount.IsOne());
+    const bool edge_is_mutable = edge->refcount.IsOne();
 
     if (height-- == 0) {
       tree->edges_[pos.index] = ResizeEdge(edge, length, edge_is_mutable);
@@ -973,7 +1004,7 @@ char CordRepBtree::GetCharacter(size_t offset) const {
 Span<char> CordRepBtree::GetAppendBufferSlow(size_t size) {
   // The inlined version in `GetAppendBuffer()` deals with all heights <= 3.
   assert(height() >= 4);
-  assert(refcount.IsMutable());
+  assert(refcount.IsOne());
 
   // Build a stack of nodes we may potentially need to update if we find a
   // non-shared FLAT with capacity at the leaf level.
@@ -982,13 +1013,13 @@ Span<char> CordRepBtree::GetAppendBufferSlow(size_t size) {
   CordRepBtree* stack[kMaxDepth];
   for (int i = 0; i < depth; ++i) {
     node = node->Edge(kBack)->btree();
-    if (!node->refcount.IsMutable()) return {};
+    if (!node->refcount.IsOne()) return {};
     stack[i] = node;
   }
 
   // Must be a privately owned, mutable flat.
   CordRep* const edge = node->Edge(kBack);
-  if (!edge->refcount.IsMutable() || edge->tag < FLAT) return {};
+  if (!edge->refcount.IsOne() || edge->tag < FLAT) return {};
 
   // Must have capacity.
   const size_t avail = edge->flat()->Capacity() - edge->length;
@@ -1123,6 +1154,79 @@ CordRepBtree* CordRepBtree::Rebuild(CordRepBtree* tree) {
   return nullptr;
 }
 
+CordRepBtree::ExtractResult CordRepBtree::ExtractAppendBuffer(
+    CordRepBtree* tree, size_t extra_capacity) {
+  int depth = 0;
+  NodeStack stack;
+
+  // Set up default 'no success' result which is {tree, nullptr}.
+  ExtractResult result;
+  result.tree = tree;
+  result.extracted = nullptr;
+
+  // Dive down the right side of the tree, making sure no edges are shared.
+  while (tree->height() > 0) {
+    if (!tree->refcount.IsOne()) return result;
+    stack[depth++] = tree;
+    tree = tree->Edge(kBack)->btree();
+  }
+  if (!tree->refcount.IsOne()) return result;
+
+  // Validate we ended on a non shared flat.
+  CordRep* rep = tree->Edge(kBack);
+  if (!(rep->IsFlat() && rep->refcount.IsOne())) return result;
+
+  // Verify it has at least the requested extra capacity.
+  CordRepFlat* flat = rep->flat();
+  const size_t length = flat->length;
+  const size_t avail = flat->Capacity() - flat->length;
+  if (extra_capacity > avail) return result;
+
+  // Set the extracted flat in the result.
+  result.extracted = flat;
+
+  // Cascading delete all nodes that become empty.
+  while (tree->size() == 1) {
+    CordRepBtree::Delete(tree);
+    if (--depth < 0) {
+      // We consumed the entire tree: return nullptr for new tree.
+      result.tree = nullptr;
+      return result;
+    }
+    rep = tree;
+    tree = stack[depth];
+  }
+
+  // Remove the edge or cascaded up parent node.
+  tree->set_end(tree->end() - 1);
+  tree->length -= length;
+
+  // Adjust lengths up the tree.
+  while (depth > 0) {
+    tree = stack[--depth];
+    tree->length -= length;
+  }
+
+  // Remove unnecessary top nodes with size = 1. This may iterate all the way
+  // down to the leaf node in which case we simply return the remaining last
+  // edge in that node and the extracted flat.
+  while (tree->size() == 1) {
+    int height = tree->height();
+    rep = tree->Edge(kBack);
+    Delete(tree);
+    if (height == 0) {
+      // We consumed the leaf: return the sole data edge as the new tree.
+      result.tree = rep;
+      return result;
+    }
+    tree = rep->btree();
+  }
+
+  // Done: return the (new) top level node and extracted flat.
+  result.tree = tree;
+  return result;
+}
+
 }  // namespace cord_internal
 ABSL_NAMESPACE_END
 }  // namespace absl

data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.h

@@ -22,6 +22,7 @@
 #include "absl/base/config.h"
 #include "absl/base/internal/raw_logging.h"
 #include "absl/base/optimization.h"
+#include "absl/strings/internal/cord_data_edge.h"
 #include "absl/strings/internal/cord_internal.h"
 #include "absl/strings/internal/cord_rep_flat.h"
 #include "absl/strings/string_view.h"
@@ -94,8 +95,9 @@ class CordRepBtree : public CordRep {
   // local stack variable compared to Cord's current near 400 bytes stack use.
   // The maximum `height` value of a node is then `kMaxDepth - 1` as node height
   // values start with a value of 0 for leaf nodes.
-  static constexpr int kMaxDepth = 12;
-  static constexpr int kMaxHeight = kMaxDepth - 1;
+  static constexpr size_t kMaxDepth = 12;
+  // See comments on height() for why this is an int and not a size_t.
+  static constexpr int kMaxHeight = static_cast<int>(kMaxDepth - 1);
 
   // `Action` defines the action for unwinding changes done at the btree's leaf
   // level that need to be propagated up to the parent node(s). Each operation
@@ -163,6 +165,9 @@ class CordRepBtree : public CordRep {
   // typically after a ref_count.Decrement() on the last reference count.
   static void Destroy(CordRepBtree* tree);
 
+  // Destruction
+  static void Delete(CordRepBtree* tree) { delete tree; }
+
   // Use CordRep::Unref() as we overload for absl::Span<CordRep* const>.
   using CordRep::Unref;
 
@@ -240,11 +245,41 @@ class CordRepBtree : public CordRep {
   // length of the flat node and involved tree nodes have been increased by
   // `span.length()`. The caller is responsible for immediately assigning values
   // to all uninitialized data reference by the returned span.
-  // Requires `this->refcount.IsMutable()`: this function forces the
-  // caller to do this fast path check on the top level node, as this is the
-  // most commonly shared node of a cord tree.
+  // Requires `this->refcount.IsOne()`: this function forces the caller to do
+  // this fast path check on the top level node, as this is the most commonly
+  // shared node of a cord tree.
   Span<char> GetAppendBuffer(size_t size);
 
+  // Extracts the right-most data edge from this tree iff:
+  // - the tree and all internal edges to the right-most node are not shared.
+  // - the right-most node is a FLAT node and not shared.
+  // - the right-most node has at least the desired extra capacity.
+  //
+  // Returns {tree, nullptr} if any of the above conditions are not met.
+  // This method effectively removes data from the tree. The intent of this
+  // method is to allow applications appending small string data to use
+  // pre-existing capacity, and add the modified rep back to the tree.
+  //
+  // Simplified such code would look similar to this:
+  //   void MyTreeBuilder::Append(string_view data) {
+  //     ExtractResult result = CordRepBtree::ExtractAppendBuffer(tree_, 1);
+  //     if (CordRep* rep = result.extracted) {
+  //       size_t available = rep->Capacity() - rep->length;
+  //       size_t n = std::min(data.size(), n);
+  //       memcpy(rep->Data(), data.data(), n);
+  //       rep->length += n;
+  //       data.remove_prefix(n);
+  //       if (!result.tree->IsBtree()) {
+  //         tree_ = CordRepBtree::Create(result.tree);
+  //       }
+  //       tree_ = CordRepBtree::Append(tree_, rep);
+  //     }
+  //     ...
+  //     // Remaining edge in `result.tree`.
+  //   }
+  static ExtractResult ExtractAppendBuffer(CordRepBtree* tree,
+                                           size_t extra_capacity = 1);
+
   // Returns the `height` of the tree. The height of a tree is limited to
   // kMaxHeight. `height` is implemented as an `int` as in some places we
   // use negative (-1) values for 'data edges'.
@@ -277,13 +312,6 @@ class CordRepBtree : public CordRep {
   // Requires this instance to be a leaf node, and `index` to be valid index.
   inline absl::string_view Data(size_t index) const;
 
-  static const char* EdgeDataPtr(const CordRep* r);
-  static absl::string_view EdgeData(const CordRep* r);
-
-  // Returns true if the provided rep is a FLAT, EXTERNAL or a SUBSTRING node
-  // holding a FLAT or EXTERNAL child rep.
-  static bool IsDataEdge(const CordRep* rep);
-
   // Diagnostics: returns true if `tree` is valid and internally consistent.
   // If `shallow` is false, then the provided top level node and all child nodes
   // below it are recursively checked. If `shallow` is true, only the provided
@@ -410,12 +438,6 @@ class CordRepBtree : public CordRep {
   // Requires `offset` < length.
   Position IndexBeyond(size_t offset) const;
 
-  // Destruction
-  static void DestroyLeaf(CordRepBtree* tree, size_t begin, size_t end);
-  static void DestroyNonLeaf(CordRepBtree* tree, size_t begin, size_t end);
-  static void DestroyTree(CordRepBtree* tree, size_t begin, size_t end);
-  static void Delete(CordRepBtree* tree) { delete tree; }
-
   // Creates a new leaf node containing as much data as possible from `data`.
   // The data is added either forwards or reversed depending on `edge_type`.
   // Callers must check the length of the returned node to determine if all data
@@ -424,9 +446,9 @@ class CordRepBtree : public CordRep {
   template <EdgeType edge_type>
   static CordRepBtree* NewLeaf(absl::string_view data, size_t extra);
 
-  // Creates a raw copy of this Btree node, copying all properties, but
-  // without adding any references to existing edges.
-  CordRepBtree* CopyRaw() const;
+  // Creates a raw copy of this Btree node with the specified length, copying
+  // all properties, but without adding any references to existing edges.
+  CordRepBtree* CopyRaw(size_t new_length) const;
 
   // Creates a full copy of this Btree node, adding a reference on all edges.
   CordRepBtree* Copy() const;
@@ -604,34 +626,11 @@ inline absl::Span<CordRep* const> CordRepBtree::Edges(size_t begin,
   return {edges_ + begin, static_cast<size_t>(end - begin)};
 }
 
-inline const char* CordRepBtree::EdgeDataPtr(const CordRep* r) {
-  assert(IsDataEdge(r));
-  size_t offset = 0;
-  if (r->tag == SUBSTRING) {
-    offset = r->substring()->start;
-    r = r->substring()->child;
-  }
-  return (r->tag >= FLAT ? r->flat()->Data() : r->external()->base) + offset;
-}
-
-inline absl::string_view CordRepBtree::EdgeData(const CordRep* r) {
-  return absl::string_view(EdgeDataPtr(r), r->length);
-}
-
 inline absl::string_view CordRepBtree::Data(size_t index) const {
   assert(height() == 0);
   return EdgeData(Edge(index));
 }
 
-inline bool CordRepBtree::IsDataEdge(const CordRep* rep) {
-  // The fast path is that `rep` is an EXTERNAL or FLAT node, making the below
-  // if a single, well predicted branch. We then repeat the FLAT or EXTERNAL
-  // check in the slow path the SUBSTRING check to optimize for the hot path.
-  if (rep->tag == EXTERNAL || rep->tag >= FLAT) return true;
-  if (rep->tag == SUBSTRING) rep = rep->substring()->child;
-  return rep->tag == EXTERNAL || rep->tag >= FLAT;
-}
-
 inline CordRepBtree* CordRepBtree::New(int height) {
   CordRepBtree* tree = new CordRepBtree;
   tree->length = 0;
@@ -659,19 +658,6 @@ inline CordRepBtree* CordRepBtree::New(CordRepBtree* front,
   return tree;
 }
 
-inline void CordRepBtree::DestroyTree(CordRepBtree* tree, size_t begin,
-                                      size_t end) {
-  if (tree->height() == 0) {
-    DestroyLeaf(tree, begin, end);
-  } else {
-    DestroyNonLeaf(tree, begin, end);
-  }
-}
-
-inline void CordRepBtree::Destroy(CordRepBtree* tree) {
-  DestroyTree(tree, tree->begin(), tree->end());
-}
-
 inline void CordRepBtree::Unref(absl::Span<CordRep* const> edges) {
   for (CordRep* edge : edges) {
     if (ABSL_PREDICT_FALSE(!edge->refcount.Decrement())) {
@@ -680,15 +666,28 @@ inline void CordRepBtree::Unref(absl::Span<CordRep* const> edges) {
   }
 }
 
-inline CordRepBtree* CordRepBtree::CopyRaw() const {
-  auto* tree = static_cast<CordRepBtree*>(::operator new(sizeof(CordRepBtree)));
-  memcpy(static_cast<void*>(tree), this, sizeof(CordRepBtree));
-  new (&tree->refcount) RefcountAndFlags;
+inline CordRepBtree* CordRepBtree::CopyRaw(size_t new_length) const {
+  CordRepBtree* tree = new CordRepBtree;
+
+  // `length` and `refcount` are the first members of `CordRepBtree`.
+  // We initialize `length` using the given length, have `refcount` be set to
+  // ref = 1 through its default constructor, and copy all data beyond
+  // 'refcount' which starts with `tag` using a single memcpy: all contents
+  // except `refcount` is trivially copyable, and the compiler does not
+  // efficiently coalesce member-wise copy of these members.
+  // See https://gcc.godbolt.org/z/qY8zsca6z
+  // # LINT.IfChange(copy_raw)
+  tree->length = new_length;
+  uint8_t* dst = &tree->tag;
+  const uint8_t* src = &tag;
+  const ptrdiff_t offset = src - reinterpret_cast<const uint8_t*>(this);
+  memcpy(dst, src, sizeof(CordRepBtree) - static_cast<size_t>(offset));
   return tree;
+  // # LINT.ThenChange()
 }
 
 inline CordRepBtree* CordRepBtree::Copy() const {
-  CordRepBtree* tree = CopyRaw();
+  CordRepBtree* tree = CopyRaw(length);
   for (CordRep* rep : Edges()) CordRep::Ref(rep);
   return tree;
 }
@@ -697,8 +696,7 @@ inline CordRepBtree* CordRepBtree::CopyToEndFrom(size_t begin,
                                                  size_t new_length) const {
   assert(begin >= this->begin());
   assert(begin <= this->end());
-  CordRepBtree* tree = CopyRaw();
-  tree->length = new_length;
+  CordRepBtree* tree = CopyRaw(new_length);
   tree->set_begin(begin);
   for (CordRep* edge : tree->Edges()) CordRep::Ref(edge);
   return tree;
@@ -708,8 +706,7 @@ inline CordRepBtree* CordRepBtree::CopyBeginTo(size_t end,
                                                size_t new_length) const {
   assert(end <= capacity());
   assert(end >= this->begin());
-  CordRepBtree* tree = CopyRaw();
-  tree->length = new_length;
+  CordRepBtree* tree = CopyRaw(new_length);
   tree->set_end(end);
   for (CordRep* edge : tree->Edges()) CordRep::Ref(edge);
   return tree;
@@ -731,7 +728,7 @@ inline void CordRepBtree::AlignBegin() {
     // size, and then do overlapping load/store of up to 4 pointers (inlined as
     // XMM, YMM or ZMM load/store) and up to 2 pointers (XMM / YMM), which is a)
     // compact and b) not clobbering any registers.
-    ABSL_INTERNAL_ASSUME(new_end <= kMaxCapacity);
+    ABSL_ASSUME(new_end <= kMaxCapacity);
 #ifdef __clang__
 #pragma unroll 1
 #endif
@@ -749,7 +746,7 @@ inline void CordRepBtree::AlignEnd() {
     const size_t new_end = end() + delta;
     set_begin(new_begin);
     set_end(new_end);
-    ABSL_INTERNAL_ASSUME(new_end <= kMaxCapacity);
+    ABSL_ASSUME(new_end <= kMaxCapacity);
 #ifdef __clang__
 #pragma unroll 1
 #endif
@@ -849,7 +846,7 @@ inline CordRepBtree* CordRepBtree::Create(CordRep* rep) {
 }
 
 inline Span<char> CordRepBtree::GetAppendBuffer(size_t size) {
-  assert(refcount.IsMutable());
+  assert(refcount.IsOne());
   CordRepBtree* tree = this;
   const int height = this->height();
   CordRepBtree* n1 = tree;
@@ -858,21 +855,21 @@ inline Span<char> CordRepBtree::GetAppendBuffer(size_t size) {
   switch (height) {
     case 3:
       tree = tree->Edge(kBack)->btree();
-      if (!tree->refcount.IsMutable()) return {};
+      if (!tree->refcount.IsOne()) return {};
       n2 = tree;
       ABSL_FALLTHROUGH_INTENDED;
     case 2:
       tree = tree->Edge(kBack)->btree();
-      if (!tree->refcount.IsMutable()) return {};
+      if (!tree->refcount.IsOne()) return {};
       n1 = tree;
       ABSL_FALLTHROUGH_INTENDED;
     case 1:
       tree = tree->Edge(kBack)->btree();
-      if (!tree->refcount.IsMutable()) return {};
+      if (!tree->refcount.IsOne()) return {};
       ABSL_FALLTHROUGH_INTENDED;
     case 0:
       CordRep* edge = tree->Edge(kBack);
-      if (!edge->refcount.IsMutable()) return {};
+      if (!edge->refcount.IsOne()) return {};
       if (edge->tag < FLAT) return {};
       size_t avail = edge->flat()->Capacity() - edge->length;
       if (avail == 0) return {};