grpc 1.46.3-x86_64-linux → 1.47.0-x86_64-linux

data/src/core/lib/promise/sleep.cc

@@ -16,6 +16,8 @@
 
 #include "src/core/lib/promise/sleep.h"
 
+#include "src/core/lib/iomgr/exec_ctx.h"
+
 namespace grpc_core {
 
 Sleep::Sleep(Timestamp deadline) : state_(new State(deadline)) {

data/src/core/lib/promise/sleep.h

@@ -17,6 +17,16 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <utility>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/status/status.h"
+
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/gprpp/time.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/poll.h"

data/src/core/lib/promise/try_seq.h

@@ -17,12 +17,12 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <tuple>
+#include <type_traits>
 #include <utility>
 
+#include "absl/meta/type_traits.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
-#include "absl/types/variant.h"
 
 #include "src/core/lib/promise/detail/basic_seq.h"
 #include "src/core/lib/promise/detail/status.h"

data/src/core/lib/resolver/resolver_factory.h

@@ -25,11 +25,10 @@
 
 #include "src/core/lib/gprpp/memory.h"
 #include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/iomgr/iomgr_fwd.h"
 #include "src/core/lib/resolver/resolver.h"
 #include "src/core/lib/uri/uri_parser.h"
 
-typedef struct grpc_pollset_set grpc_pollset_set;
-
 namespace grpc_core {
 
 // TODO(yashkt): Move WorkSerializer to its own Bazel target, depend on that

data/src/core/lib/resolver/server_address.cc

@@ -25,6 +25,7 @@
 #include <vector>
 
 #include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 
 #include "src/core/lib/address_utils/sockaddr_utils.h"
@@ -80,6 +81,7 @@ ServerAddress::ServerAddress(ServerAddress&& other) noexcept
       attributes_(std::move(other.attributes_)) {
   other.args_ = nullptr;
 }
+
 ServerAddress& ServerAddress::operator=(ServerAddress&& other) noexcept {
   address_ = other.address_;
   grpc_channel_args_destroy(args_);
@@ -149,12 +151,12 @@ ServerAddress ServerAddress::WithAttribute(
 }
 
 std::string ServerAddress::ToString() const {
+  auto addr_str = grpc_sockaddr_to_string(&address_, false);
   std::vector<std::string> parts = {
-      grpc_sockaddr_to_string(&address_, false),
+      addr_str.ok() ? addr_str.value() : addr_str.status().ToString(),
   };
   if (args_ != nullptr) {
-    parts.emplace_back(
-        absl::StrCat("args={", grpc_channel_args_string(args_), "}"));
+    parts.emplace_back(absl::StrCat("args=", grpc_channel_args_string(args_)));
   }
   if (!attributes_.empty()) {
     std::vector<std::string> attrs;
@@ -167,4 +169,8 @@ std::string ServerAddress::ToString() const {
   return absl::StrJoin(parts, " ");
 }
 
+std::string ServerAddressWeightAttribute::ToString() const {
+  return absl::StrFormat("%d", weight_);
+}
+
 }  // namespace grpc_core

data/src/core/lib/resolver/server_address.h

@@ -25,7 +25,6 @@
 #include <memory>
 
 #include "absl/container/inlined_vector.h"
-#include "absl/strings/str_format.h"
 
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/gpr/useful.h"
@@ -96,6 +95,9 @@ class ServerAddress {
   ServerAddress WithAttribute(const char* key,
                               std::unique_ptr<AttributeInterface> value) const;
 
+  // TODO(ctiller): Prior to making this a public API we should ensure that the
+  // channel args are not part of the generated string, lest we make that debug
+  // format load-bearing via Hyrum's law.
   std::string ToString() const;
 
  private:
@@ -131,9 +133,7 @@ class ServerAddressWeightAttribute : public ServerAddress::AttributeInterface {
     return QsortCompare(weight_, other_locality_attr->weight_);
   }
 
-  std::string ToString() const override {
-    return absl::StrFormat("%d", weight_);
-  }
+  std::string ToString() const override;
 
  private:
   uint32_t weight_;

data/src/core/lib/resource_quota/api.cc

@@ -16,11 +16,24 @@
 
 #include "src/core/lib/resource_quota/api.h"
 
+#include <stdint.h>
+
+#include <atomic>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "absl/strings/str_cat.h"
+
 #include <grpc/grpc.h>
 
-#include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/channel_args_preconditioning.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/resource_quota/memory_quota.h"
 #include "src/core/lib/resource_quota/resource_quota.h"
+#include "src/core/lib/resource_quota/thread_quota.h"
 
 namespace grpc_core {
 

data/src/core/lib/resource_quota/api.h

@@ -17,7 +17,10 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/lib/channel/channel_args.h"
+#include <stddef.h>
+
+#include <grpc/impl/codegen/grpc_types.h>
+
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/resource_quota/resource_quota.h"
 

data/src/core/lib/resource_quota/arena.cc

@@ -20,17 +20,11 @@
 
 #include "src/core/lib/resource_quota/arena.h"
 
-#include <string.h>
-
 #include <new>
 
 #include <grpc/support/alloc.h>
-#include <grpc/support/atm.h>
-#include <grpc/support/log.h>
-#include <grpc/support/sync.h>
 
 #include "src/core/lib/gpr/alloc.h"
-#include "src/core/lib/gprpp/memory.h"
 
 namespace {
 

data/src/core/lib/resource_quota/arena.h

@@ -34,8 +34,7 @@
 #include <new>
 #include <utility>
 
-#include <grpc/support/alloc.h>
-#include <grpc/support/sync.h>
+#include <grpc/event_engine/memory_allocator.h>
 
 #include "src/core/lib/gpr/alloc.h"
 #include "src/core/lib/promise/context.h"

data/src/core/lib/resource_quota/memory_quota.cc

@@ -16,8 +16,18 @@
 
 #include "src/core/lib/resource_quota/memory_quota.h"
 
+#include <inttypes.h>
+
+#include <algorithm>
 #include <atomic>
+#include <tuple>
+#include <type_traits>
+
+#include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "absl/utility/utility.h"
 
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/mpscq.h"
 #include "src/core/lib/promise/exec_ctx_wakeup_scheduler.h"
@@ -25,7 +35,6 @@
 #include "src/core/lib/promise/map.h"
 #include "src/core/lib/promise/race.h"
 #include "src/core/lib/promise/seq.h"
-#include "src/core/lib/resource_quota/memory_quota.h"
 #include "src/core/lib/resource_quota/trace.h"
 
 namespace grpc_core {
@@ -183,7 +192,9 @@ size_t GrpcMemoryAllocatorImpl::Reserve(MemoryRequest request) {
   while (true) {
     // Attempt to reserve memory from our pool.
     auto reservation = TryReserve(request);
-    if (reservation.has_value()) return *reservation;
+    if (reservation.has_value()) {
+      return *reservation;
+    }
     // If that failed, grab more from the quota and retry.
     Replenish();
   }
@@ -241,6 +252,28 @@ absl::optional<size_t> GrpcMemoryAllocatorImpl::TryReserve(
   }
 }
 
+void GrpcMemoryAllocatorImpl::MaybeDonateBack() {
+  size_t free = free_bytes_.load(std::memory_order_relaxed);
+  const size_t kReduceToSize = kMaxQuotaBufferSize / 2;
+  while (true) {
+    if (free <= kReduceToSize) return;
+    size_t ret = free - kReduceToSize;
+    if (free_bytes_.compare_exchange_weak(free, kReduceToSize,
+                                          std::memory_order_acq_rel,
+                                          std::memory_order_acquire)) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
+        gpr_log(GPR_INFO, "[%p|%s] Early return %" PRIdPTR " bytes", this,
+                name_.c_str(), ret);
+      }
+      MutexLock lock(&memory_quota_mu_);
+      GPR_ASSERT(taken_bytes_ >= ret);
+      taken_bytes_ -= ret;
+      memory_quota_->Return(ret);
+      return;
+    }
+  }
+}
+
 void GrpcMemoryAllocatorImpl::Replenish() {
   MutexLock lock(&memory_quota_mu_);
   GPR_ASSERT(!shutdown_);
@@ -376,8 +409,12 @@ void BasicMemoryQuota::Start() {
           std::tuple<const char*, RefCountedPtr<ReclaimerQueue::Handle>> arg) {
         auto reclaimer = std::move(std::get<1>(arg));
         if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
-          gpr_log(GPR_INFO, "RQ: %s perform %s reclamation",
-                  self->name_.c_str(), std::get<0>(arg));
+          double free = std::max(intptr_t(0), self->free_bytes_.load());
+          size_t quota_size = self->quota_size_.load();
+          gpr_log(GPR_INFO,
+                  "RQ: %s perform %s reclamation. Available free bytes: %f, "
+                  "total quota_size: %zu",
+                  self->name_.c_str(), std::get<0>(arg), free, quota_size);
         }
         // One of the reclaimer queues gave us a way to get back memory.
         // Call the reclaimer with a token that contains enough to wake us
@@ -436,7 +473,12 @@ void BasicMemoryQuota::FinishReclamation(uint64_t token, Waker waker) {
                                                    std::memory_order_relaxed,
                                                    std::memory_order_relaxed)) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
-      gpr_log(GPR_INFO, "RQ: %s reclamation complete", name_.c_str());
+      double free = std::max(intptr_t(0), free_bytes_.load());
+      size_t quota_size = quota_size_.load();
+      gpr_log(GPR_INFO,
+              "RQ: %s reclamation complete. Available free bytes: %f, "
+              "total quota_size: %zu",
+              name_.c_str(), free, quota_size);
     }
     waker.Wakeup();
   }

data/src/core/lib/resource_quota/memory_quota.h

@@ -17,18 +17,25 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <algorithm>
+#include <stdint.h>
+
 #include <atomic>
 #include <cstddef>
 #include <limits>
 #include <memory>
-#include <queue>
-#include <vector>
+#include <string>
+#include <utility>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
 
 #include <grpc/event_engine/memory_allocator.h>
-#include <grpc/slice.h>
+#include <grpc/event_engine/memory_request.h>
+#include <grpc/support/log.h>
 
 #include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/poll.h"
@@ -73,6 +80,7 @@ enum class ReclamationPass {
   kDestructive = 3,
 };
 static constexpr size_t kNumReclamationPasses = 4;
+static constexpr size_t kMaxQuotaBufferSize = 1024 * 1024;
 
 // For each reclamation function run we construct a ReclamationSweep.
 // When this object is finally destroyed (it may be moved several times first),
@@ -295,7 +303,12 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl {
     // Add the released memory to our free bytes counter... if this increases
     // from  0 to non-zero, then we have more to do, otherwise, we're actually
     // done.
-    if (free_bytes_.fetch_add(n, std::memory_order_release) != 0) return;
+    size_t prev_free = free_bytes_.fetch_add(n, std::memory_order_release);
+    if (prev_free + n > kMaxQuotaBufferSize) {
+      // Try to immediately return some free'ed memory back to the total quota.
+      MaybeDonateBack();
+    }
+    if (prev_free != 0) return;
     MaybeRegisterReclaimer();
   }
 
@@ -323,6 +336,12 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl {
  private:
   // Primitive reservation function.
   absl::optional<size_t> TryReserve(MemoryRequest request) GRPC_MUST_USE_RESULT;
+  // This function may be invoked during a memory release operation. If the
+  // total free_bytes in this allocator/local cache exceeds
+  // kMaxQuotaBufferSize / 2, donate the excess free_bytes in this cache back
+  // to the total quota immediately. This helps prevent free bytes in any
+  // particular allocator from growing too large.
+  void MaybeDonateBack();
   // Replenish bytes from the quota, without blocking, possibly entering
   // overcommit.
   void Replenish() ABSL_LOCKS_EXCLUDED(memory_quota_mu_);

data/src/core/lib/resource_quota/resource_quota.h

@@ -17,16 +17,24 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <string>
+#include <utility>
+
+#include "absl/strings/string_view.h"
+
 #include <grpc/impl/codegen/grpc_types.h>
 
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/cpp_impl_of.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/resource_quota/memory_quota.h"
 #include "src/core/lib/resource_quota/thread_quota.h"
 
 namespace grpc_core {
 
 class ResourceQuota;
+
 using ResourceQuotaRefPtr = RefCountedPtr<ResourceQuota>;
 
 class ResourceQuota : public RefCounted<ResourceQuota>,

data/src/core/lib/resource_quota/thread_quota.cc

@@ -16,6 +16,8 @@
 
 #include "src/core/lib/resource_quota/thread_quota.h"
 
+#include <grpc/support/log.h>
+
 namespace grpc_core {
 
 ThreadQuota::ThreadQuota() = default;

data/src/core/lib/resource_quota/thread_quota.h

@@ -18,8 +18,12 @@
 #include <grpc/support/port_platform.h>
 
 #include <cstddef>
+#include <limits>
+
+#include "absl/base/thread_annotations.h"
 
 #include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
 
 namespace grpc_core {

data/src/core/lib/security/authorization/grpc_server_authz_filter.cc

@@ -16,7 +16,10 @@
 
 #include "src/core/lib/security/authorization/grpc_server_authz_filter.h"
 
+#include "absl/strings/str_join.h"
+
 #include "src/core/lib/channel/promise_based_filter.h"
+#include "src/core/lib/promise/promise.h"
 #include "src/core/lib/security/authorization/evaluate_args.h"
 #include "src/core/lib/transport/transport.h"
 

data/src/core/lib/security/credentials/alts/alts_credentials.cc

@@ -56,7 +56,10 @@ grpc_alts_credentials::create_security_connector(
       this->Ref(), std::move(call_creds), target_name);
 }
 
-const char* grpc_alts_credentials::type() const { return "Alts"; }
+grpc_core::UniqueTypeName grpc_alts_credentials::type() const {
+  static grpc_core::UniqueTypeName::Factory kFactory("Alts");
+  return kFactory.Create();
+}
 
 grpc_alts_server_credentials::grpc_alts_server_credentials(
     const grpc_alts_credentials_options* options,
@@ -79,7 +82,10 @@ grpc_alts_server_credentials::~grpc_alts_server_credentials() {
   gpr_free(handshaker_service_url_);
 }
 
-const char* grpc_alts_server_credentials::type() const { return "Alts"; }
+grpc_core::UniqueTypeName grpc_alts_server_credentials::type() const {
+  static grpc_core::UniqueTypeName::Factory kFactory("Alts");
+  return kFactory.Create();
+}
 
 grpc_channel_credentials* grpc_alts_credentials_create_customized(
     const grpc_alts_credentials_options* options,

data/src/core/lib/security/credentials/alts/alts_credentials.h

@@ -39,7 +39,7 @@ class grpc_alts_credentials final : public grpc_channel_credentials {
       const char* target_name, const grpc_channel_args* args,
       grpc_channel_args** new_args) override;
 
-  const char* type() const override;
+  grpc_core::UniqueTypeName type() const override;
 
   const grpc_alts_credentials_options* options() const { return options_; }
   grpc_alts_credentials_options* mutable_options() { return options_; }
@@ -66,7 +66,7 @@ class grpc_alts_server_credentials final : public grpc_server_credentials {
   grpc_core::RefCountedPtr<grpc_server_security_connector>
   create_security_connector(const grpc_channel_args* /* args */) override;
 
-  const char* type() const override;
+  grpc_core::UniqueTypeName type() const override;
 
   const grpc_alts_credentials_options* options() const { return options_; }
   grpc_alts_credentials_options* mutable_options() { return options_; }

data/src/core/lib/security/credentials/call_creds_util.cc

@@ -21,6 +21,8 @@
 #include "absl/strings/str_cat.h"
 #include "absl/strings/string_view.h"
 
+#include <grpc/support/string_util.h>
+
 namespace grpc_core {
 
 namespace {

data/src/core/lib/security/credentials/channel_creds_registry.h

@@ -21,6 +21,7 @@
 
 #include <grpc/impl/codegen/grpc_types.h>
 
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/json/json.h"
 
 struct grpc_channel_credentials;

data/src/core/lib/security/credentials/composite/composite_credentials.cc

@@ -41,8 +41,9 @@
 // grpc_composite_channel_credentials
 //
 
-const char* grpc_composite_channel_credentials::type() const {
-  return "Composite";
+grpc_core::UniqueTypeName grpc_composite_channel_credentials::type() const {
+  static grpc_core::UniqueTypeName::Factory kFactory("Composite");
+  return kFactory.Create();
 }
 
 /* -- Composite call credentials. -- */
@@ -60,7 +61,10 @@ grpc_composite_call_credentials::GetRequestMetadata(
       });
 }
 
-const char* grpc_composite_call_credentials::Type() { return "Composite"; }
+grpc_core::UniqueTypeName grpc_composite_call_credentials::Type() {
+  static grpc_core::UniqueTypeName::Factory kFactory("Composite");
+  return kFactory.Create();
+}
 
 std::string grpc_composite_call_credentials::debug_string() {
   std::vector<std::string> outputs;

data/src/core/lib/security/credentials/composite/composite_credentials.h

@@ -51,11 +51,12 @@ class grpc_composite_channel_credentials : public grpc_channel_credentials {
       const char* target, const grpc_channel_args* args,
       grpc_channel_args** new_args) override;
 
-  grpc_channel_args* update_arguments(grpc_channel_args* args) override {
+  grpc_core::ChannelArgs update_arguments(
+      grpc_core::ChannelArgs args) override {
     return inner_creds_->update_arguments(args);
   }
 
-  const char* type() const override;
+  grpc_core::UniqueTypeName type() const override;
 
   const grpc_channel_credentials* inner_creds() const {
     return inner_creds_.get();
@@ -98,9 +99,9 @@ class grpc_composite_call_credentials : public grpc_call_credentials {
   const CallCredentialsList& inner() const { return inner_; }
   std::string debug_string() override;
 
-  static const char* Type();
+  static grpc_core::UniqueTypeName Type();
 
-  const char* type() const override { return Type(); }
+  grpc_core::UniqueTypeName type() const override { return Type(); }
 
  private:
   int cmp_impl(const grpc_call_credentials* other) const override {

data/src/core/lib/security/credentials/credentials.h

@@ -29,7 +29,9 @@
 #include <grpc/grpc_security.h>
 #include <grpc/support/sync.h>
 
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/unique_type_name.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/promise/arena_promise.h"
 #include "src/core/lib/security/context/security_context.h"
@@ -92,6 +94,15 @@ void grpc_override_well_known_credentials_path_getter(
 struct grpc_channel_credentials
     : grpc_core::RefCounted<grpc_channel_credentials> {
  public:
+  static absl::string_view ChannelArgName() {
+    return GRPC_ARG_CHANNEL_CREDENTIALS;
+  }
+
+  static int ChannelArgsCompare(const grpc_channel_credentials* args1,
+                                const grpc_channel_credentials* args2) {
+    return args1->cmp(args2);
+  }
+
   // Creates a security connector for the channel. May also create new channel
   // args for the channel to be used in place of the passed in const args if
   // returned non NULL. In that case the caller is responsible for destroying
@@ -115,7 +126,7 @@ struct grpc_channel_credentials
   // By default, leave channel args as is. The callee takes ownership
   // of the passed-in channel args, and the caller takes ownership
   // of the returned channel args.
-  virtual grpc_channel_args* update_arguments(grpc_channel_args* args) {
+  virtual grpc_core::ChannelArgs update_arguments(grpc_core::ChannelArgs args) {
     return args;
   }
 
@@ -129,9 +140,7 @@ struct grpc_channel_credentials
   // as equal (assuming other channel args match).
   int cmp(const grpc_channel_credentials* other) const {
     GPR_ASSERT(other != nullptr);
-    // Intentionally uses grpc_core::QsortCompare instead of strcmp as a safety
-    // against different grpc_channel_credentials types using the same name.
-    int r = grpc_core::QsortCompare(type(), other->type());
+    int r = type().Compare(other->type());
     if (r != 0) return r;
     return cmp_impl(other);
   }
@@ -140,7 +149,7 @@ struct grpc_channel_credentials
   // implementation for down-casting purposes. Every creds implementation should
   // use a unique string instance, which should be returned by all instances of
   // that creds implementation.
-  virtual const char* type() const = 0;
+  virtual grpc_core::UniqueTypeName type() const = 0;
 
  private:
   // Implementation for `cmp` method intended to be overridden by subclasses.
@@ -212,9 +221,7 @@ struct grpc_call_credentials
   // credentials as effectively the same..
   int cmp(const grpc_call_credentials* other) const {
     GPR_ASSERT(other != nullptr);
-    // Intentionally uses grpc_core::QsortCompare instead of strcmp as a safety
-    // against different grpc_call_credentials types using the same name.
-    int r = grpc_core::QsortCompare(type(), other->type());
+    int r = type().Compare(other->type());
     if (r != 0) return r;
     return cmp_impl(other);
   }
@@ -227,7 +234,7 @@ struct grpc_call_credentials
   // implementation for down-casting purposes. Every creds implementation should
   // use a unique string instance, which should be returned by all instances of
   // that creds implementation.
-  virtual const char* type() const = 0;
+  virtual grpc_core::UniqueTypeName type() const = 0;
 
  private:
   // Implementation for `cmp` method intended to be overridden by subclasses.
@@ -256,7 +263,7 @@ struct grpc_server_credentials
   virtual grpc_core::RefCountedPtr<grpc_server_security_connector>
   create_security_connector(const grpc_channel_args* args) = 0;
 
-  virtual const char* type() const = 0;
+  virtual grpc_core::UniqueTypeName type() const = 0;
 
   const grpc_auth_metadata_processor& auth_metadata_processor() const {
     return processor_;