grpc 1.42.0 → 1.43.1

data/src/core/lib/slice/slice.h

@@ -0,0 +1,341 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_SLICE_SLICE_H
+#define GRPC_CORE_LIB_SLICE_SLICE_H
+
+#include <grpc/support/port_platform.h>
+
+#include "absl/strings/string_view.h"
+
+#include <grpc/slice.h>
+
+#include "src/core/lib/slice/slice_internal.h"
+
+// Herein lies grpc_core::Slice and its team of thin wrappers around grpc_slice.
+// They aim to keep you safe by providing strong guarantees around lifetime and
+// mutability.
+//
+// The team:
+//   Slice        - provides a wrapper around an unknown type of slice.
+//                  Immutable (since we don't know who else might be referencing
+//                  it), and potentially ref counted.
+//   StaticSlice  - provides a wrapper around a static slice. Not refcounted,
+//                  fast to copy.
+//   MutableSlice - provides a guarantee of unique ownership, meaning the
+//                  underlying data can be mutated safely.
+
+namespace grpc_core {
+
+// Forward declarations
+class Slice;
+class StaticSlice;
+class MutableSlice;
+
+namespace slice_detail {
+
+// Returns an empty slice.
+static constexpr grpc_slice EmptySlice() { return {nullptr, {}}; }
+
+// BaseSlice holds the grpc_slice object, but does not apply refcounting policy.
+// It does export immutable access into the slice, such that this can be shared
+// by all storage policies.
+class BaseSlice {
+ public:
+  BaseSlice(const BaseSlice&) = delete;
+  BaseSlice& operator=(const BaseSlice&) = delete;
+  BaseSlice(BaseSlice&& other) = delete;
+  BaseSlice& operator=(BaseSlice&& other) = delete;
+
+  // Iterator access to the underlying bytes
+  const uint8_t* begin() const { return GRPC_SLICE_START_PTR(c_slice()); }
+  const uint8_t* end() const { return GRPC_SLICE_END_PTR(c_slice()); }
+  const uint8_t* cbegin() const { return GRPC_SLICE_START_PTR(c_slice()); }
+  const uint8_t* cend() const { return GRPC_SLICE_END_PTR(c_slice()); }
+
+  // Retrieve a borrowed reference to the underlying grpc_slice.
+  const grpc_slice& c_slice() const { return slice_; }
+
+  // Retrieve the underlying grpc_slice, and replace the one in this object with
+  // EmptySlice().
+  grpc_slice TakeCSlice() {
+    grpc_slice out = slice_;
+    slice_ = EmptySlice();
+    return out;
+  }
+
+  // As other things... borrowed references.
+  absl::string_view as_string_view() const {
+    return absl::string_view(reinterpret_cast<const char*>(data()), size());
+  }
+
+  // Array access
+  uint8_t operator[](size_t i) const {
+    return GRPC_SLICE_START_PTR(c_slice())[i];
+  }
+
+  // Access underlying data
+  const uint8_t* data() const { return GRPC_SLICE_START_PTR(c_slice()); }
+
+  // Size of the slice
+  size_t size() const { return GRPC_SLICE_LENGTH(c_slice()); }
+  size_t length() const { return size(); }
+  bool empty() const { return size() == 0; }
+
+  // For inlined slices - are these two slices equal?
+  // For non-inlined slices - do these two slices refer to the same block of
+  // memory?
+  bool is_equivalent(const BaseSlice& other) const {
+    return grpc_slice_is_equivalent(slice_, other.slice_);
+  }
+
+ protected:
+  BaseSlice() : slice_(EmptySlice()) {}
+  explicit BaseSlice(const grpc_slice& slice) : slice_(slice) {}
+  ~BaseSlice() = default;
+
+  void Swap(BaseSlice* other) { std::swap(slice_, other->slice_); }
+  void SetCSlice(const grpc_slice& slice) { slice_ = slice; }
+
+  uint8_t* mutable_data() { return GRPC_SLICE_START_PTR(slice_); }
+
+ private:
+  grpc_slice slice_;
+};
+
+inline bool operator==(const BaseSlice& a, const BaseSlice& b) {
+  return grpc_slice_eq(a.c_slice(), b.c_slice()) != 0;
+}
+
+inline bool operator!=(const BaseSlice& a, const BaseSlice& b) {
+  return grpc_slice_eq(a.c_slice(), b.c_slice()) == 0;
+}
+
+inline bool operator==(const BaseSlice& a, absl::string_view b) {
+  return a.as_string_view() == b;
+}
+
+inline bool operator!=(const BaseSlice& a, absl::string_view b) {
+  return a.as_string_view() != b;
+}
+
+inline bool operator==(absl::string_view a, const BaseSlice& b) {
+  return a == b.as_string_view();
+}
+
+inline bool operator!=(absl::string_view a, const BaseSlice& b) {
+  return a != b.as_string_view();
+}
+
+inline bool operator==(const BaseSlice& a, const grpc_slice& b) {
+  return grpc_slice_eq(a.c_slice(), b) != 0;
+}
+
+inline bool operator!=(const BaseSlice& a, const grpc_slice& b) {
+  return grpc_slice_eq(a.c_slice(), b) == 0;
+}
+
+inline bool operator==(const grpc_slice& a, const BaseSlice& b) {
+  return grpc_slice_eq(a, b.c_slice()) != 0;
+}
+
+inline bool operator!=(const grpc_slice& a, const BaseSlice& b) {
+  return grpc_slice_eq(a, b.c_slice()) == 0;
+}
+
+template <typename Out>
+struct CopyConstructors {
+  static Out FromCopiedString(const char* s) {
+    return Out(grpc_slice_from_copied_string(s));
+  }
+  static Out FromCopiedString(std::string s) {
+    return Out(grpc_slice_from_cpp_string(std::move(s)));
+  }
+  static Out FromCopiedBuffer(const char* p, size_t len) {
+    return Out(UnmanagedMemorySlice(p, len));
+  }
+
+  template <typename Buffer>
+  static Out FromCopiedBuffer(const Buffer& buffer) {
+    return FromCopiedBuffer(reinterpret_cast<const char*>(buffer.data()),
+                            buffer.size());
+  }
+};
+
+}  // namespace slice_detail
+
+class StaticSlice : public slice_detail::BaseSlice {
+ public:
+  StaticSlice() = default;
+  explicit StaticSlice(const grpc_slice& slice)
+      : slice_detail::BaseSlice(slice) {
+    GPR_DEBUG_ASSERT(
+        slice.refcount->GetType() == grpc_slice_refcount::Type::STATIC ||
+        slice.refcount->GetType() == grpc_slice_refcount::Type::NOP);
+  }
+  explicit StaticSlice(const StaticMetadataSlice& slice)
+      : slice_detail::BaseSlice(slice) {}
+
+  static StaticSlice FromStaticString(const char* s) {
+    return StaticSlice(grpc_slice_from_static_string(s));
+  }
+
+  static StaticSlice FromStaticString(absl::string_view s) {
+    return StaticSlice(ExternallyManagedSlice(s.data(), s.size()));
+  }
+
+  StaticSlice(const StaticSlice& other)
+      : slice_detail::BaseSlice(other.c_slice()) {}
+  StaticSlice& operator=(const StaticSlice& other) {
+    SetCSlice(other.c_slice());
+    return *this;
+  }
+  StaticSlice(StaticSlice&& other) noexcept
+      : slice_detail::BaseSlice(other.TakeCSlice()) {}
+  StaticSlice& operator=(StaticSlice&& other) noexcept {
+    Swap(&other);
+    return *this;
+  }
+};
+
+class MutableSlice : public slice_detail::BaseSlice,
+                     public slice_detail::CopyConstructors<MutableSlice> {
+ public:
+  MutableSlice() = default;
+  explicit MutableSlice(const grpc_slice& slice)
+      : slice_detail::BaseSlice(slice) {
+    GPR_DEBUG_ASSERT(slice.refcount == nullptr ||
+                     slice.refcount->IsRegularUnique());
+  }
+  ~MutableSlice() { grpc_slice_unref_internal(c_slice()); }
+
+  MutableSlice(const MutableSlice&) = delete;
+  MutableSlice& operator=(const MutableSlice&) = delete;
+  MutableSlice(MutableSlice&& other) noexcept
+      : slice_detail::BaseSlice(other.TakeCSlice()) {}
+  MutableSlice& operator=(MutableSlice&& other) noexcept {
+    Swap(&other);
+    return *this;
+  }
+
+  static MutableSlice CreateUninitialized(size_t length) {
+    return MutableSlice(grpc_slice_malloc(length));
+  }
+
+  // Return a sub slice of this one. Leaves this slice in an indeterminate but
+  // valid state.
+  MutableSlice TakeSubSlice(size_t pos, size_t n) {
+    return MutableSlice(grpc_slice_sub_no_ref(TakeCSlice(), pos, pos + n));
+  }
+
+  // Iterator access to the underlying bytes
+  uint8_t* begin() { return mutable_data(); }
+  uint8_t* end() { return mutable_data() + size(); }
+
+  // Array access
+  uint8_t& operator[](size_t i) { return mutable_data()[i]; }
+};
+
+class Slice : public slice_detail::BaseSlice,
+              public slice_detail::CopyConstructors<Slice> {
+ public:
+  Slice() = default;
+  ~Slice() { grpc_slice_unref_internal(c_slice()); }
+  explicit Slice(const grpc_slice& slice) : slice_detail::BaseSlice(slice) {}
+  explicit Slice(slice_detail::BaseSlice&& other)
+      : slice_detail::BaseSlice(other.TakeCSlice()) {}
+
+  Slice(const Slice&) = delete;
+  Slice& operator=(const Slice&) = delete;
+  Slice(Slice&& other) noexcept : slice_detail::BaseSlice(other.TakeCSlice()) {}
+  Slice& operator=(Slice&& other) noexcept {
+    Swap(&other);
+    return *this;
+  }
+
+  // A slice might refer to some memory that we keep a refcount to (this is
+  // owned), or some memory that's inlined into the slice (also owned), or some
+  // other block of memory that we know will be available for the lifetime of
+  // some operation in the common case (not owned). In the *less common* case
+  // that we need to keep that slice text for longer than our API's guarantee us
+  // access, we need to take a copy and turn this into something that we do own.
+
+  // TakeOwned returns an owned slice regardless of current ownership, and
+  // leaves the current slice in a valid but externally unpredictable state - in
+  // doing so it can avoid adding a ref to the underlying slice.
+  Slice TakeOwned() {
+    if (c_slice().refcount == nullptr) {
+      return Slice(c_slice());
+    }
+    if (c_slice().refcount->GetType() == grpc_slice_refcount::Type::NOP) {
+      return Slice(grpc_slice_copy(c_slice()));
+    }
+    return Slice(TakeCSlice());
+  }
+
+  // AsOwned returns an owned slice but does not mutate the current slice,
+  // meaning that it may add a reference to the underlying slice.
+  Slice AsOwned() const {
+    if (c_slice().refcount == nullptr) {
+      return Slice(c_slice());
+    }
+    if (c_slice().refcount->GetType() == grpc_slice_refcount::Type::NOP) {
+      return Slice(grpc_slice_copy(c_slice()));
+    }
+    return Slice(grpc_slice_ref_internal(c_slice()));
+  }
+
+  // TakeMutable returns a MutableSlice, and leaves the current slice in an
+  // indeterminate but valid state.
+  // A mutable slice requires only one reference to the bytes of the slice -
+  // this can be achieved either with inlined storage or with a single
+  // reference.
+  // If the current slice is refcounted and there are more than one references
+  // to that slice, then the slice is copied in order to achieve a mutable
+  // version.
+  MutableSlice TakeMutable() {
+    if (c_slice().refcount == nullptr) {
+      return MutableSlice(c_slice());
+    }
+    if (c_slice().refcount->GetType() == grpc_slice_refcount::Type::REGULAR &&
+        c_slice().refcount->IsRegularUnique()) {
+      return MutableSlice(TakeCSlice());
+    }
+    return MutableSlice(grpc_slice_copy(c_slice()));
+  }
+
+  // Return a sub slice of this one. Leaves this slice in an indeterminate but
+  // valid state.
+  Slice TakeSubSlice(size_t pos, size_t n) {
+    return Slice(grpc_slice_sub_no_ref(TakeCSlice(), pos, pos + n));
+  }
+
+  Slice Ref() const { return Slice(grpc_slice_ref_internal(c_slice())); }
+
+  Slice Copy() const { return Slice(grpc_slice_copy(c_slice())); }
+
+  static Slice FromRefcountAndBytes(grpc_slice_refcount* r,
+                                    const uint8_t* begin, const uint8_t* end) {
+    grpc_slice out;
+    out.refcount = r;
+    r->Ref();
+    out.data.refcounted.bytes = const_cast<uint8_t*>(begin);
+    out.data.refcounted.length = end - begin;
+    return Slice(out);
+  }
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_LIB_SLICE_SLICE_H

data/src/core/lib/slice/slice_buffer.cc

@@ -84,6 +84,10 @@ void grpc_slice_buffer_destroy_internal(grpc_slice_buffer* sb) {
   grpc_slice_buffer_reset_and_unref_internal(sb);
   if (sb->base_slices != sb->inlined) {
     gpr_free(sb->base_slices);
+    // As a precaution, set sb->base_slices to equal sb->inlined
+    // to prevent a double free attempt if grpc_slice_buffer_destroy_internal
+    // is invoked two times on the same slice buffer.
+    sb->base_slices = sb->slices = sb->inlined;
   }
 }
 

data/src/core/lib/slice/slice_intern.cc

@@ -235,7 +235,7 @@ static InternedSliceRefcount* MatchInternedSliceLocked(uint32_t hash,
   /* search for an existing string */
   for (s = shard->strs[idx]; s; s = s->bucket_next) {
     if (s->hash == hash && grpc_core::InternedSlice(s) == args) {
-      if (s->refcnt.RefIfNonZero()) {
+      if (grpc_core::IncrementIfNonzero(&s->refcnt)) {
         return s;
       }
     }

data/src/core/lib/slice/slice_refcount.h

@@ -17,6 +17,8 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <grpc/support/alloc.h>
+
 #include "src/core/lib/slice/slice_refcount_base.h"
 #include "src/core/lib/slice/static_slice.h"
 
@@ -28,6 +30,8 @@ namespace grpc_core {
 
 extern grpc_slice_refcount kNoopRefcount;
 
+// TODO(ctiller): when this is removed, remove the std::atomic* in
+// grpc_slice_refcount and just put it there directly.
 struct InternedSliceRefcount {
   static void Destroy(void* arg) {
     auto* rc = static_cast<InternedSliceRefcount*>(arg);
@@ -48,7 +52,7 @@ struct InternedSliceRefcount {
   grpc_slice_refcount base;
   grpc_slice_refcount sub;
   const size_t length;
-  RefCount refcnt;
+  std::atomic<size_t> refcnt{1};
   const uint32_t hash;
   InternedSliceRefcount* bucket_next;
 };

data/src/core/lib/slice/slice_refcount_base.h

@@ -17,9 +17,10 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <grpc/slice.h>
+#include <atomic>
 
-#include "src/core/lib/gprpp/ref_counted.h"
+#include <grpc/slice.h>
+#include <grpc/support/log.h>
 
 // grpc_slice_refcount : A reference count for grpc_slice.
 //
@@ -103,11 +104,10 @@ struct grpc_slice_refcount {
   //  Whether we are the refcount for a static
   //  metadata slice, an interned slice, or any other kind of slice.
   //
-  //  2. RefCount* ref
-  //  The pointer to the actual underlying grpc_core::RefCount. Rather than
-  //  performing struct offset computations as in the original implementation to
-  //  get to the refcount, which requires a virtual method, we devirtualize by
-  //  using a nullable pointer to allow a single pair of Ref/Unref methods.
+  //  2. std::atomic<size_t>* ref
+  //  The pointer to the actual underlying grpc_core::RefCount.
+  //  TODO(ctiller): remove the pointer indirection and just put the refcount on
+  //  this object once we remove interning.
   //
   //  3. DestroyerFn destroyer_fn
   //  Called when the refcount goes to 0, with destroyer_arg as parameter.
@@ -117,7 +117,7 @@ struct grpc_slice_refcount {
   //
   //  5. grpc_slice_refcount* sub
   //  Argument used for interned slices.
-  grpc_slice_refcount(grpc_slice_refcount::Type type, grpc_core::RefCount* ref,
+  grpc_slice_refcount(grpc_slice_refcount::Type type, std::atomic<size_t>* ref,
                       DestroyerFn destroyer_fn, void* destroyer_arg,
                       grpc_slice_refcount* sub)
       : ref_(ref),
@@ -136,19 +136,27 @@ struct grpc_slice_refcount {
   uint32_t Hash(const grpc_slice& slice);
   void Ref() {
     if (ref_ == nullptr) return;
-    ref_->RefNonZero();
+    ref_->fetch_add(1, std::memory_order_relaxed);
   }
   void Unref() {
     if (ref_ == nullptr) return;
-    if (ref_->Unref()) {
+    if (ref_->fetch_sub(1, std::memory_order_acq_rel) == 1) {
       dest_fn_(destroy_fn_arg_);
     }
   }
 
+  // Only for type REGULAR, is this the only instance?
+  // For this to be useful the caller needs to ensure that if this is the only
+  // instance, no other instance could be created during this call.
+  bool IsRegularUnique() {
+    GPR_DEBUG_ASSERT(ref_type_ == Type::REGULAR);
+    return ref_->load(std::memory_order_relaxed) == 1;
+  }
+
   grpc_slice_refcount* sub_refcount() const { return sub_refcount_; }
 
  private:
-  grpc_core::RefCount* ref_ = nullptr;
+  std::atomic<size_t>* ref_ = nullptr;
   const Type ref_type_ = Type::REGULAR;
   grpc_slice_refcount* sub_refcount_ = this;
   DestroyerFn dest_fn_ = nullptr;