grpc 1.42.0 → 1.51.0

data/src/core/lib/slice/slice.cc

@@ -18,15 +18,19 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "src/core/lib/slice/slice.h"
+
 #include <string.h>
 
+#include <new>
+
 #include <grpc/slice.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
 #include "src/core/lib/gprpp/memory.h"
-#include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/slice/slice_refcount.h"
 
 char* grpc_slice_to_c_string(grpc_slice slice) {
   char* out = static_cast<char*>(gpr_malloc(GRPC_SLICE_LENGTH(slice) + 1));
@@ -35,7 +39,9 @@ char* grpc_slice_to_c_string(grpc_slice slice) {
   return out;
 }
 
-grpc_slice grpc_empty_slice(void) { return grpc_core::UnmanagedMemorySlice(); }
+grpc_slice grpc_empty_slice(void) {
+  return grpc_core::slice_detail::EmptySlice();
+}
 
 grpc_slice grpc_slice_copy(grpc_slice s) {
   grpc_slice out = GRPC_SLICE_MALLOC(GRPC_SLICE_LENGTH(s));
@@ -46,30 +52,21 @@ grpc_slice grpc_slice_copy(grpc_slice s) {
 
 namespace grpc_core {
 
-/* grpc_slice_from_static_string support structure - a refcount that does
-   nothing */
-grpc_slice_refcount kNoopRefcount(grpc_slice_refcount::Type::NOP);
-static_assert(std::is_trivially_destructible<decltype(kNoopRefcount)>::value,
-              "kNoopRefcount must be trivially destructible.");
-
 /* grpc_slice_new support structures - we create a refcount object extended
    with the user provided data pointer & destroy function */
-class NewSliceRefcount {
+class NewSliceRefcount : public grpc_slice_refcount {
  public:
-  static void Destroy(void* arg) { delete static_cast<NewSliceRefcount*>(arg); }
-
   NewSliceRefcount(void (*destroy)(void*), void* user_data)
-      : base_(grpc_slice_refcount::Type::REGULAR, &refs_, Destroy, this,
-              &base_),
+      : grpc_slice_refcount(Destroy),
         user_destroy_(destroy),
         user_data_(user_data) {}
   ~NewSliceRefcount() { user_destroy_(user_data_); }
 
-  grpc_slice_refcount* base_refcount() { return &base_; }
-
  private:
-  grpc_slice_refcount base_;
-  RefCount refs_;
+  static void Destroy(grpc_slice_refcount* arg) {
+    delete static_cast<NewSliceRefcount*>(arg);
+  }
+
   void (*user_destroy_)(void*);
   void* user_data_;
 };
@@ -77,7 +74,8 @@ class NewSliceRefcount {
 }  // namespace grpc_core
 
 size_t grpc_slice_memory_usage(grpc_slice s) {
-  if (s.refcount == nullptr || s.refcount == &grpc_core::kNoopRefcount) {
+  if (s.refcount == nullptr ||
+      s.refcount == grpc_slice_refcount::NoopRefcount()) {
     return 0;
   } else {
     return s.data.refcounted.length;
@@ -85,19 +83,18 @@ size_t grpc_slice_memory_usage(grpc_slice s) {
 }
 
 grpc_slice grpc_slice_from_static_buffer(const void* s, size_t len) {
-  return grpc_core::ExternallyManagedSlice(s, len);
+  return grpc_core::StaticSlice::FromStaticBuffer(s, len).TakeCSlice();
 }
 
 grpc_slice grpc_slice_from_static_string(const char* s) {
-  return grpc_core::ExternallyManagedSlice(s, strlen(s));
+  return grpc_core::StaticSlice::FromStaticString(s).TakeCSlice();
 }
 
 grpc_slice grpc_slice_new_with_user_data(void* p, size_t len,
                                          void (*destroy)(void*),
                                          void* user_data) {
   grpc_slice slice;
-  slice.refcount =
-      (new grpc_core::NewSliceRefcount(destroy, user_data))->base_refcount();
+  slice.refcount = new grpc_core::NewSliceRefcount(destroy, user_data);
   slice.data.refcounted.bytes = static_cast<uint8_t*>(p);
   slice.data.refcounted.length = len;
   return slice;
@@ -111,68 +108,57 @@ grpc_slice grpc_slice_new(void* p, size_t len, void (*destroy)(void*)) {
 namespace grpc_core {
 /* grpc_slice_new_with_len support structures - we create a refcount object
    extended with the user provided data pointer & destroy function */
-class NewWithLenSliceRefcount {
+class NewWithLenSliceRefcount : public grpc_slice_refcount {
  public:
-  static void Destroy(void* arg) {
-    delete static_cast<NewWithLenSliceRefcount*>(arg);
-  }
-
   NewWithLenSliceRefcount(void (*destroy)(void*, size_t), void* user_data,
                           size_t user_length)
-      : base_(grpc_slice_refcount::Type::REGULAR, &refs_, Destroy, this,
-              &base_),
+      : grpc_slice_refcount(Destroy),
         user_data_(user_data),
         user_length_(user_length),
         user_destroy_(destroy) {}
   ~NewWithLenSliceRefcount() { user_destroy_(user_data_, user_length_); }
 
-  grpc_slice_refcount* base_refcount() { return &base_; }
-
  private:
-  grpc_slice_refcount base_;
-  RefCount refs_;
+  static void Destroy(grpc_slice_refcount* arg) {
+    delete static_cast<NewWithLenSliceRefcount*>(arg);
+  }
+
   void* user_data_;
   size_t user_length_;
   void (*user_destroy_)(void*, size_t);
 };
 
 /** grpc_slice_from_moved_(string|buffer) ref count .*/
-class MovedStringSliceRefCount {
+class MovedStringSliceRefCount : public grpc_slice_refcount {
  public:
-  explicit MovedStringSliceRefCount(grpc_core::UniquePtr<char>&& str)
-      : base_(grpc_slice_refcount::Type::REGULAR, &refs_, Destroy, this,
-              &base_),
-        str_(std::move(str)) {}
-
-  grpc_slice_refcount* base_refcount() { return &base_; }
+  explicit MovedStringSliceRefCount(UniquePtr<char>&& str)
+      : grpc_slice_refcount(Destroy), str_(std::move(str)) {}
 
  private:
-  static void Destroy(void* arg) {
+  static void Destroy(grpc_slice_refcount* arg) {
     delete static_cast<MovedStringSliceRefCount*>(arg);
   }
 
-  grpc_slice_refcount base_;
-  grpc_core::RefCount refs_;
-  grpc_core::UniquePtr<char> str_;
+  UniquePtr<char> str_;
 };
 
 // grpc_slice_from_cpp_string() ref count.
-class MovedCppStringSliceRefCount {
+class MovedCppStringSliceRefCount : public grpc_slice_refcount {
  public:
   explicit MovedCppStringSliceRefCount(std::string&& str)
-      : base_(grpc_slice_refcount::Type::REGULAR, &refs_, Destroy, this,
-              &base_),
-        str_(std::move(str)) {}
+      : grpc_slice_refcount(Destroy), str_(std::move(str)) {}
+
+  uint8_t* data() {
+    return reinterpret_cast<uint8_t*>(const_cast<char*>(str_.data()));
+  }
 
-  grpc_slice_refcount* base_refcount() { return &base_; }
+  size_t size() const { return str_.size(); }
 
  private:
-  static void Destroy(void* arg) {
+  static void Destroy(grpc_slice_refcount* arg) {
     delete static_cast<MovedCppStringSliceRefCount*>(arg);
   }
 
-  grpc_slice_refcount base_;
-  grpc_core::RefCount refs_;
   std::string str_;
 };
 
@@ -181,42 +167,17 @@ class MovedCppStringSliceRefCount {
 grpc_slice grpc_slice_new_with_len(void* p, size_t len,
                                    void (*destroy)(void*, size_t)) {
   grpc_slice slice;
-  slice.refcount = (new grpc_core::NewWithLenSliceRefcount(destroy, p, len))
-                       ->base_refcount();
+  slice.refcount = new grpc_core::NewWithLenSliceRefcount(destroy, p, len);
   slice.data.refcounted.bytes = static_cast<uint8_t*>(p);
   slice.data.refcounted.length = len;
   return slice;
 }
 
-grpc_core::UnmanagedMemorySlice::UnmanagedMemorySlice(const char* source,
-                                                      size_t length) {
-  if (length <= sizeof(data.inlined.bytes)) {
-    refcount = nullptr;
-    data.inlined.length = static_cast<uint8_t>(length);
-  } else {
-    HeapInit(length);
-  }
-  if (length > 0) {
-    memcpy(GRPC_SLICE_START_PTR(*this), source, length);
-  }
-}
-
-grpc_core::UnmanagedMemorySlice::UnmanagedMemorySlice(const char* source)
-    : grpc_core::UnmanagedMemorySlice::UnmanagedMemorySlice(source,
-                                                            strlen(source)) {}
-
-grpc_slice grpc_slice_from_copied_buffer(const char* source, size_t length) {
-  grpc_slice slice;
-  if (length <= sizeof(slice.data.inlined.bytes)) {
-    slice.refcount = nullptr;
-    slice.data.inlined.length = length;
-  } else {
-    // Create a ref-counted slice.
-    slice = grpc_core::UnmanagedMemorySlice(
-        length, grpc_core::UnmanagedMemorySlice::ForceHeapAllocation());
-  }
-  memcpy(GRPC_SLICE_START_PTR(slice), source, length);
-  return slice;
+grpc_slice grpc_slice_from_copied_buffer(const char* source, size_t len) {
+  if (len == 0) return grpc_empty_slice();
+  grpc_slice out = grpc_slice_malloc(len);
+  memcpy(GRPC_SLICE_START_PTR(out), source, len);
+  return out;
 }
 
 grpc_slice grpc_slice_from_copied_string(const char* source) {
@@ -232,8 +193,7 @@ grpc_slice grpc_slice_from_moved_buffer(grpc_core::UniquePtr<char> p,
     slice.data.inlined.length = len;
     memcpy(GRPC_SLICE_START_PTR(slice), ptr, len);
   } else {
-    slice.refcount = (new grpc_core::MovedStringSliceRefCount(std::move(p)))
-                         ->base_refcount();
+    slice.refcount = new grpc_core::MovedStringSliceRefCount(std::move(p));
     slice.data.refcounted.bytes = ptr;
     slice.data.refcounted.length = len;
   }
@@ -252,97 +212,47 @@ grpc_slice grpc_slice_from_cpp_string(std::string str) {
     slice.data.inlined.length = str.size();
     memcpy(GRPC_SLICE_START_PTR(slice), str.data(), str.size());
   } else {
-    slice.data.refcounted.bytes =
-        reinterpret_cast<uint8_t*>(const_cast<char*>(str.data()));
-    slice.data.refcounted.length = str.size();
-    slice.refcount =
-        (new grpc_core::MovedCppStringSliceRefCount(std::move(str)))
-            ->base_refcount();
+    auto* refcount = new grpc_core::MovedCppStringSliceRefCount(std::move(str));
+    slice.data.refcounted.bytes = refcount->data();
+    slice.data.refcounted.length = refcount->size();
+    slice.refcount = refcount;
   }
   return slice;
 }
 
-namespace {
-
-class MallocRefCount {
- public:
-  static void Destroy(void* arg) {
-    MallocRefCount* r = static_cast<MallocRefCount*>(arg);
-    r->~MallocRefCount();
-    gpr_free(r);
-  }
-
-  MallocRefCount()
-      : base_(grpc_slice_refcount::Type::REGULAR, &refs_, Destroy, this,
-              &base_) {}
-  ~MallocRefCount() = default;
-
-  grpc_slice_refcount* base_refcount() { return &base_; }
-
- private:
-  grpc_slice_refcount base_;
-  grpc_core::RefCount refs_;
-};
-
-}  // namespace
-
 grpc_slice grpc_slice_malloc_large(size_t length) {
-  return grpc_core::UnmanagedMemorySlice(
-      length, grpc_core::UnmanagedMemorySlice::ForceHeapAllocation());
-}
-
-void grpc_core::UnmanagedMemorySlice::HeapInit(size_t length) {
-  /* Memory layout used by the slice created here:
-
-     +-----------+----------------------------------------------------------+
-     | refcount  | bytes                                                    |
-     +-----------+----------------------------------------------------------+
-
-     refcount is a malloc_refcount
-     bytes is an array of bytes of the requested length
-     Both parts are placed in the same allocation returned from gpr_malloc */
-  auto* rc =
-      static_cast<MallocRefCount*>(gpr_malloc(sizeof(MallocRefCount) + length));
-
-  /* Initial refcount on rc is 1 - and it's up to the caller to release
-     this reference. */
-  new (rc) MallocRefCount();
-
-  /* Build up the slice to be returned. */
-  /* The slices refcount points back to the allocated block. */
-  refcount = rc->base_refcount();
-  /* The data bytes are placed immediately after the refcount struct */
-  data.refcounted.bytes = reinterpret_cast<uint8_t*>(rc + 1);
-  /* And the length of the block is set to the requested length */
-  data.refcounted.length = length;
+  grpc_slice slice;
+  uint8_t* memory = new uint8_t[sizeof(grpc_slice_refcount) + length];
+  slice.refcount = new (memory) grpc_slice_refcount(
+      [](grpc_slice_refcount* p) { delete[] reinterpret_cast<uint8_t*>(p); });
+  slice.data.refcounted.bytes = memory + sizeof(grpc_slice_refcount);
+  slice.data.refcounted.length = length;
+  return slice;
 }
 
 grpc_slice grpc_slice_malloc(size_t length) {
-  return grpc_core::UnmanagedMemorySlice(length);
-}
-
-grpc_core::UnmanagedMemorySlice::UnmanagedMemorySlice(size_t length) {
-  if (length > sizeof(data.inlined.bytes)) {
-    HeapInit(length);
+  if (length <= GRPC_SLICE_INLINED_SIZE) {
+    grpc_slice slice;
+    slice.refcount = nullptr;
+    slice.data.inlined.length = length;
+    return slice;
   } else {
-    /* small slice: just inline the data */
-    refcount = nullptr;
-    data.inlined.length = static_cast<uint8_t>(length);
+    return grpc_slice_malloc_large(length);
   }
 }
 
-template <typename Slice>
-static Slice sub_no_ref(const Slice& source, size_t begin, size_t end) {
-  Slice subset;
+static grpc_slice sub_no_ref(const grpc_slice& source, size_t begin,
+                             size_t end) {
+  grpc_slice subset;
 
   GPR_ASSERT(end >= begin);
 
-  if (source.refcount) {
+  if (source.refcount != nullptr) {
     /* Enforce preconditions */
     GPR_ASSERT(source.data.refcounted.length >= end);
 
     /* Build the result */
-    subset.refcount = source.refcount->sub_refcount();
+    subset.refcount = source.refcount;
     /* Point into the source array */
     subset.data.refcounted.bytes = source.data.refcounted.bytes + begin;
     subset.data.refcounted.length = end - begin;
@@ -361,11 +271,6 @@ grpc_slice grpc_slice_sub_no_ref(grpc_slice source, size_t begin, size_t end) {
   return sub_no_ref(source, begin, end);
 }
 
-grpc_core::UnmanagedMemorySlice grpc_slice_sub_no_ref(
-    const grpc_core::UnmanagedMemorySlice& source, size_t begin, size_t end) {
-  return sub_no_ref(source, begin, end);
-}
-
 grpc_slice grpc_slice_sub(grpc_slice source, size_t begin, size_t end) {
   grpc_slice subset;
 
@@ -377,7 +282,9 @@ grpc_slice grpc_slice_sub(grpc_slice source, size_t begin, size_t end) {
   } else {
     subset = grpc_slice_sub_no_ref(source, begin, end);
     /* Bump the refcount */
-    subset.refcount->Ref();
+    if (subset.refcount != grpc_slice_refcount::NoopRefcount()) {
+      subset.refcount->Ref();
+    }
   }
   return subset;
 }
@@ -395,6 +302,12 @@ grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice* source, size_t split,
     memcpy(tail.data.inlined.bytes, source->data.inlined.bytes + split,
            tail.data.inlined.length);
     source->data.inlined.length = static_cast<uint8_t>(split);
+  } else if (source->refcount == grpc_slice_refcount::NoopRefcount()) {
+    /* refcount == NoopRefcount(), so we can just split in-place */
+    tail.refcount = grpc_slice_refcount::NoopRefcount();
+    tail.data.refcounted.bytes = source->data.refcounted.bytes + split;
+    tail.data.refcounted.length = source->data.refcounted.length - split;
+    source->data.refcounted.length = split;
   } else {
     size_t tail_length = source->data.refcounted.length - split;
     GPR_ASSERT(source->data.refcounted.length >= split);
@@ -405,23 +318,22 @@ grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice* source, size_t split,
       tail.data.inlined.length = static_cast<uint8_t>(tail_length);
       memcpy(tail.data.inlined.bytes, source->data.refcounted.bytes + split,
              tail_length);
-      source->refcount = source->refcount->sub_refcount();
     } else {
       /* Build the result */
       switch (ref_whom) {
         case GRPC_SLICE_REF_TAIL:
-          tail.refcount = source->refcount->sub_refcount();
-          source->refcount = &grpc_core::kNoopRefcount;
+          tail.refcount = source->refcount;
+          source->refcount = grpc_slice_refcount::NoopRefcount();
           break;
         case GRPC_SLICE_REF_HEAD:
-          tail.refcount = &grpc_core::kNoopRefcount;
-          source->refcount = source->refcount->sub_refcount();
+          tail.refcount = grpc_slice_refcount::NoopRefcount();
           break;
         case GRPC_SLICE_REF_BOTH:
-          tail.refcount = source->refcount->sub_refcount();
-          source->refcount = source->refcount->sub_refcount();
+          tail.refcount = source->refcount;
           /* Bump the refcount */
-          tail.refcount->Ref();
+          if (tail.refcount != grpc_slice_refcount::NoopRefcount()) {
+            tail.refcount->Ref();
+          }
           break;
       }
       /* Point into the source array */
@@ -457,20 +369,20 @@ grpc_slice grpc_slice_split_head(grpc_slice* source, size_t split) {
     head.refcount = nullptr;
     head.data.inlined.length = static_cast<uint8_t>(split);
     memcpy(head.data.inlined.bytes, source->data.refcounted.bytes, split);
-    source->refcount = source->refcount->sub_refcount();
     source->data.refcounted.bytes += split;
     source->data.refcounted.length -= split;
   } else {
     GPR_ASSERT(source->data.refcounted.length >= split);
 
     /* Build the result */
-    head.refcount = source->refcount->sub_refcount();
+    head.refcount = source->refcount;
     /* Bump the refcount */
-    head.refcount->Ref();
+    if (head.refcount != grpc_slice_refcount::NoopRefcount()) {
+      head.refcount->Ref();
+    }
     /* Point into the source array */
     head.data.refcounted.bytes = source->data.refcounted.bytes;
     head.data.refcounted.length = split;
-    source->refcount = source->refcount->sub_refcount();
     source->data.refcounted.bytes += split;
     source->data.refcounted.length -= split;
   }
@@ -478,21 +390,13 @@ grpc_slice grpc_slice_split_head(grpc_slice* source, size_t split) {
   return head;
 }
 
-int grpc_slice_default_eq_impl(grpc_slice a, grpc_slice b) {
+int grpc_slice_eq(grpc_slice a, grpc_slice b) {
   if (GRPC_SLICE_LENGTH(a) != GRPC_SLICE_LENGTH(b)) return false;
   if (GRPC_SLICE_LENGTH(a) == 0) return true;
   return 0 == memcmp(GRPC_SLICE_START_PTR(a), GRPC_SLICE_START_PTR(b),
                      GRPC_SLICE_LENGTH(a));
 }
 
-int grpc_slice_eq(grpc_slice a, grpc_slice b) {
-  if (a.refcount && b.refcount &&
-      a.refcount->GetType() == b.refcount->GetType()) {
-    return a.refcount->Eq(a, b);
-  }
-  return grpc_slice_default_eq_impl(a, b);
-}
-
 int grpc_slice_differs_refcounted(const grpc_slice& a,
                                   const grpc_slice& b_not_inline) {
   size_t a_len;
@@ -590,3 +494,9 @@ grpc_slice grpc_slice_dup(grpc_slice a) {
          GRPC_SLICE_LENGTH(a));
   return copy;
 }
+
+grpc_slice grpc_slice_ref(grpc_slice slice) {
+  return grpc_core::CSliceRef(slice);
+}
+
+void grpc_slice_unref(grpc_slice slice) { grpc_core::CSliceUnref(slice); }