grpc 1.51.0 → 1.52.0

data/src/core/lib/resource_quota/arena.h

@@ -1,20 +1,20 @@
-/*
- *
- * Copyright 2017 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
+//
+//
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+//
 
 // \file Arena based allocator
 // Allows very fast allocation of memory, but that memory cannot be freed until
@@ -249,4 +249,4 @@ struct ContextType<Arena> {};
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_RESOURCE_QUOTA_ARENA_H */
+#endif  // GRPC_CORE_LIB_RESOURCE_QUOTA_ARENA_H

data/src/core/lib/resource_quota/memory_quota.cc

@@ -158,7 +158,9 @@ Poll<RefCountedPtr<ReclaimerQueue::Handle>> ReclaimerQueue::PollNext() {
 GrpcMemoryAllocatorImpl::GrpcMemoryAllocatorImpl(
     std::shared_ptr<BasicMemoryQuota> memory_quota, std::string name)
     : memory_quota_(memory_quota), name_(std::move(name)) {
-  memory_quota_->Take(taken_bytes_);
+  memory_quota_->Take(
+      /*allocator=*/this, taken_bytes_);
+  memory_quota_->AddNewAllocator(this);
 }
 
 GrpcMemoryAllocatorImpl::~GrpcMemoryAllocatorImpl() {
@@ -169,6 +171,7 @@ GrpcMemoryAllocatorImpl::~GrpcMemoryAllocatorImpl() {
 }
 
 void GrpcMemoryAllocatorImpl::Shutdown() {
+  memory_quota_->RemoveAllocator(this);
   std::shared_ptr<BasicMemoryQuota> memory_quota;
   OrphanablePtr<ReclaimerQueue::Handle>
       reclamation_handles[kNumReclamationPasses];
@@ -188,12 +191,17 @@ size_t GrpcMemoryAllocatorImpl::Reserve(MemoryRequest request) {
   // inlined asserts.
   GPR_ASSERT(request.min() <= request.max());
   GPR_ASSERT(request.max() <= MemoryRequest::max_allowed_size());
+  size_t old_free = free_bytes_.load(std::memory_order_relaxed);
+
   while (true) {
     // Attempt to reserve memory from our pool.
     auto reservation = TryReserve(request);
     if (reservation.has_value()) {
+      size_t new_free = free_bytes_.load(std::memory_order_relaxed);
+      memory_quota_->MaybeMoveAllocator(this, old_free, new_free);
       return *reservation;
     }
+
     // If that failed, grab more from the quota and retry.
     Replenish();
   }
@@ -253,9 +261,7 @@ void GrpcMemoryAllocatorImpl::MaybeDonateBack() {
         free > kMaxQuotaBufferSize / 2) {
       ret = std::max(ret, free - kMaxQuotaBufferSize / 2);
     }
-    if (IsPeriodicResourceQuotaReclamationEnabled()) {
-      ret = std::max(ret, free > 8192 ? free / 2 : free);
-    }
+    ret = std::max(ret, free > 8192 ? free / 2 : free);
     const size_t new_free = free - ret;
     if (free_bytes_.compare_exchange_weak(free, new_free,
                                           std::memory_order_acq_rel,
@@ -277,40 +283,12 @@ void GrpcMemoryAllocatorImpl::Replenish() {
   auto amount = Clamp(taken_bytes_.load(std::memory_order_relaxed) / 3,
                       kMinReplenishBytes, kMaxReplenishBytes);
   // Take the requested amount from the quota.
-  memory_quota_->Take(amount);
+  memory_quota_->Take(
+      /*allocator=*/this, amount);
   // Record that we've taken it.
   taken_bytes_.fetch_add(amount, std::memory_order_relaxed);
   // Add the taken amount to the free pool.
   free_bytes_.fetch_add(amount, std::memory_order_acq_rel);
-  // See if we can add ourselves as a reclaimer.
-  MaybeRegisterReclaimer();
-}
-
-void GrpcMemoryAllocatorImpl::MaybeRegisterReclaimer() {
-  // If the reclaimer is already registered, then there's nothing to do.
-  if (registered_reclaimer_.exchange(true, std::memory_order_relaxed)) {
-    return;
-  }
-  MutexLock lock(&reclaimer_mu_);
-  if (shutdown_) return;
-  // Grab references to the things we'll need
-  auto self = shared_from_this();
-  std::weak_ptr<EventEngineMemoryAllocatorImpl> self_weak{self};
-  registered_reclaimer_ = true;
-  InsertReclaimer(0, [self_weak](absl::optional<ReclamationSweep> sweep) {
-    if (!sweep.has_value()) return;
-    auto self = self_weak.lock();
-    if (self == nullptr) return;
-    auto* p = static_cast<GrpcMemoryAllocatorImpl*>(self.get());
-    p->registered_reclaimer_.store(false, std::memory_order_relaxed);
-    // Figure out how many bytes we can return to the quota.
-    size_t return_bytes = p->free_bytes_.exchange(0, std::memory_order_acq_rel);
-    if (return_bytes == 0) return;
-    // Subtract that from our outstanding balance.
-    p->taken_bytes_.fetch_sub(return_bytes);
-    // And return them to the quota.
-    p->memory_quota_->Return(return_bytes);
-  });
 }
 
 //
@@ -323,7 +301,6 @@ class BasicMemoryQuota::WaitForSweepPromise {
                       uint64_t token)
       : memory_quota_(std::move(memory_quota)), token_(token) {}
 
-  struct Empty {};
   Poll<Empty> operator()() {
     if (memory_quota_->reclamation_counter_.load(std::memory_order_relaxed) !=
         token_) {
@@ -362,16 +339,15 @@ void BasicMemoryQuota::Start() {
             return std::make_tuple(name, std::move(f));
           };
         };
-        return Race(Map(self->reclaimers_[0].Next(), annotate("compact")),
-                    Map(self->reclaimers_[1].Next(), annotate("benign")),
-                    Map(self->reclaimers_[2].Next(), annotate("idle")),
-                    Map(self->reclaimers_[3].Next(), annotate("destructive")));
+        return Race(Map(self->reclaimers_[0].Next(), annotate("benign")),
+                    Map(self->reclaimers_[1].Next(), annotate("idle")),
+                    Map(self->reclaimers_[2].Next(), annotate("destructive")));
       },
       [self](
           std::tuple<const char*, RefCountedPtr<ReclaimerQueue::Handle>> arg) {
         auto reclaimer = std::move(std::get<1>(arg));
         if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
-          double free = std::max(intptr_t(0), self->free_bytes_.load());
+          double free = std::max(intptr_t{0}, self->free_bytes_.load());
           size_t quota_size = self->quota_size_.load();
           gpr_log(GPR_INFO,
                   "RQ: %s perform %s reclamation. Available free bytes: %f, "
@@ -412,11 +388,11 @@ void BasicMemoryQuota::SetSize(size_t new_size) {
     Return(new_size - old_size);
   } else {
     // We're shrinking the quota.
-    Take(old_size - new_size);
+    Take(/*allocator=*/nullptr, old_size - new_size);
   }
 }
 
-void BasicMemoryQuota::Take(size_t amount) {
+void BasicMemoryQuota::Take(GrpcMemoryAllocatorImpl* allocator, size_t amount) {
   // If there's a request for nothing, then do nothing!
   if (amount == 0) return;
   GPR_DEBUG_ASSERT(amount <= std::numeric_limits<intptr_t>::max());
@@ -426,6 +402,25 @@ void BasicMemoryQuota::Take(size_t amount) {
   if (prior >= 0 && prior < static_cast<intptr_t>(amount)) {
     if (reclaimer_activity_ != nullptr) reclaimer_activity_->ForceWakeup();
   }
+
+  if (IsFreeLargeAllocatorEnabled()) {
+    if (allocator == nullptr) return;
+    GrpcMemoryAllocatorImpl* chosen_allocator = nullptr;
+    // Use calling allocator's shard index to choose shard.
+    auto& shard = big_allocators_.shards[allocator->IncrementShardIndex() %
+                                         big_allocators_.shards.size()];
+
+    if (shard.shard_mu.TryLock()) {
+      if (!shard.allocators.empty()) {
+        chosen_allocator = *shard.allocators.begin();
+      }
+      shard.shard_mu.Unlock();
+    }
+
+    if (chosen_allocator != nullptr) {
+      chosen_allocator->ReturnFree();
+    }
+  }
 }
 
 void BasicMemoryQuota::FinishReclamation(uint64_t token, Waker waker) {
@@ -435,7 +430,7 @@ void BasicMemoryQuota::FinishReclamation(uint64_t token, Waker waker) {
                                                    std::memory_order_relaxed,
                                                    std::memory_order_relaxed)) {
     if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
-      double free = std::max(intptr_t(0), free_bytes_.load());
+      double free = std::max(intptr_t{0}, free_bytes_.load());
       size_t quota_size = quota_size_.load();
       gpr_log(GPR_INFO,
               "RQ: %s reclamation complete. Available free bytes: %f, "
@@ -450,6 +445,107 @@ void BasicMemoryQuota::Return(size_t amount) {
   free_bytes_.fetch_add(amount, std::memory_order_relaxed);
 }
 
+void BasicMemoryQuota::AddNewAllocator(GrpcMemoryAllocatorImpl* allocator) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
+    gpr_log(GPR_INFO, "Adding allocator %p", allocator);
+  }
+
+  AllocatorBucket::Shard& shard = small_allocators_.SelectShard(allocator);
+
+  {
+    absl::MutexLock l(&shard.shard_mu);
+    shard.allocators.emplace(allocator);
+  }
+}
+
+void BasicMemoryQuota::RemoveAllocator(GrpcMemoryAllocatorImpl* allocator) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
+    gpr_log(GPR_INFO, "Removing allocator %p", allocator);
+  }
+
+  AllocatorBucket::Shard& small_shard =
+      small_allocators_.SelectShard(allocator);
+
+  {
+    absl::MutexLock l(&small_shard.shard_mu);
+    if (small_shard.allocators.erase(allocator) == 1) {
+      return;
+    }
+  }
+
+  AllocatorBucket::Shard& big_shard = big_allocators_.SelectShard(allocator);
+
+  {
+    absl::MutexLock l(&big_shard.shard_mu);
+    big_shard.allocators.erase(allocator);
+  }
+}
+
+void BasicMemoryQuota::MaybeMoveAllocator(GrpcMemoryAllocatorImpl* allocator,
+                                          size_t old_free_bytes,
+                                          size_t new_free_bytes) {
+  while (true) {
+    if (new_free_bytes < kSmallAllocatorThreshold) {
+      // Still in small bucket. No move.
+      if (old_free_bytes < kSmallAllocatorThreshold) return;
+      MaybeMoveAllocatorBigToSmall(allocator);
+    } else if (new_free_bytes > kBigAllocatorThreshold) {
+      // Still in big bucket. No move.
+      if (old_free_bytes > kBigAllocatorThreshold) return;
+      MaybeMoveAllocatorSmallToBig(allocator);
+    } else {
+      // Somewhere between thresholds. No move.
+      return;
+    }
+
+    // Loop to make sure move is eventually stable.
+    old_free_bytes = new_free_bytes;
+    new_free_bytes = allocator->GetFreeBytes();
+  }
+}
+
+void BasicMemoryQuota::MaybeMoveAllocatorBigToSmall(
+    GrpcMemoryAllocatorImpl* allocator) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
+    gpr_log(GPR_INFO, "Moving allocator %p to small", allocator);
+  }
+
+  AllocatorBucket::Shard& old_shard = big_allocators_.SelectShard(allocator);
+
+  {
+    absl::MutexLock l(&old_shard.shard_mu);
+    if (old_shard.allocators.erase(allocator) == 0) return;
+  }
+
+  AllocatorBucket::Shard& new_shard = small_allocators_.SelectShard(allocator);
+
+  {
+    absl::MutexLock l(&new_shard.shard_mu);
+    new_shard.allocators.emplace(allocator);
+  }
+}
+
+void BasicMemoryQuota::MaybeMoveAllocatorSmallToBig(
+    GrpcMemoryAllocatorImpl* allocator) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
+    gpr_log(GPR_INFO, "Moving allocator %p to big", allocator);
+  }
+
+  AllocatorBucket::Shard& old_shard = small_allocators_.SelectShard(allocator);
+
+  {
+    absl::MutexLock l(&old_shard.shard_mu);
+    if (old_shard.allocators.erase(allocator) == 0) return;
+  }
+
+  AllocatorBucket::Shard& new_shard = big_allocators_.SelectShard(allocator);
+
+  {
+    absl::MutexLock l(&new_shard.shard_mu);
+    new_shard.allocators.emplace(allocator);
+  }
+}
+
 BasicMemoryQuota::PressureInfo BasicMemoryQuota::GetPressureInfo() {
   double free = free_bytes_.load();
   if (free < 0) free = 0;

data/src/core/lib/resource_quota/memory_quota.h

@@ -19,6 +19,7 @@
 
 #include <stdint.h>
 
+#include <array>
 #include <atomic>
 #include <cstddef>
 #include <limits>
@@ -27,14 +28,18 @@
 #include <utility>
 
 #include "absl/base/thread_annotations.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/strings/string_view.h"
+#include "absl/synchronization/mutex.h"
 #include "absl/types/optional.h"
 
 #include <grpc/event_engine/memory_allocator.h>
 #include <grpc/event_engine/memory_request.h>
 #include <grpc/support/log.h>
 
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/experiments/experiments.h"
+#include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
@@ -42,11 +47,13 @@
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/poll.h"
 #include "src/core/lib/resource_quota/periodic_update.h"
+#include "src/core/lib/resource_quota/trace.h"
 
 namespace grpc_core {
 
 class BasicMemoryQuota;
 class MemoryQuota;
+class GrpcMemoryAllocatorImpl;
 
 using grpc_event_engine::experimental::MemoryRequest;
 
@@ -63,26 +70,22 @@ using Vector = grpc_event_engine::experimental::Vector<T>;
 // reclaimers. We do this in multiple passes: if there is a less destructive
 // operation available, we do that, otherwise we do something more destructive.
 enum class ReclamationPass {
-  // Non-empty reclamation ought to take index 0, but to simplify API we don't
-  // expose that publicly (it's an internal detail), and hence index zero is
-  // here unnamed.
-
   // Benign reclamation is intended for reclamation steps that are not
   // observable outside of gRPC (besides maybe causing an increase in CPU
   // usage).
   // Examples of such reclamation would be resizing buffers to fit the current
   // load needs, rather than whatever was the peak usage requirement.
-  kBenign = 1,
+  kBenign = 0,
   // Idle reclamation is intended for reclamation steps that are observable
   // outside of gRPC, but do not cause application work to be lost.
   // Examples of such reclamation would be dropping channels that are not being
   // used.
-  kIdle = 2,
+  kIdle = 1,
   // Destructive reclamation is our last resort, and is these reclamations are
   // allowed to drop work - such as cancelling in flight requests.
-  kDestructive = 3,
+  kDestructive = 2,
 };
-static constexpr size_t kNumReclamationPasses = 4;
+static constexpr size_t kNumReclamationPasses = 3;
 
 // For each reclamation function run we construct a ReclamationSweep.
 // When this object is finally destroyed (it may be moved several times first),
@@ -274,18 +277,24 @@ class PressureTracker {
 };
 }  // namespace memory_quota_detail
 
+// Minimum number of free bytes in order for allocator to move to big bucket.
+static constexpr size_t kBigAllocatorThreshold = 0.5 * 1024 * 1024;
+// Maximum number of free bytes in order for allocator to move to small
+// bucket.
+static constexpr size_t kSmallAllocatorThreshold = 0.1 * 1024 * 1024;
+
 class BasicMemoryQuota final
     : public std::enable_shared_from_this<BasicMemoryQuota> {
  public:
   // Data about current memory pressure.
   struct PressureInfo {
     // The current instantaneously measured memory pressure.
-    double instantaneous_pressure;
+    double instantaneous_pressure = 0.0;
     // A control value that can be used to scale buffer sizes up or down to
     // adjust memory pressure to our target set point.
-    double pressure_control_value;
+    double pressure_control_value = 0.0;
     // Maximum recommended individual allocation size.
-    size_t max_recommended_allocation_size;
+    size_t max_recommended_allocation_size = 0;
   };
 
   explicit BasicMemoryQuota(std::string name) : name_(std::move(name)) {}
@@ -302,11 +311,18 @@ class BasicMemoryQuota final
   void SetSize(size_t new_size);
   // Forcefully take some memory from the quota, potentially entering
   // overcommit.
-  void Take(size_t amount);
+  void Take(GrpcMemoryAllocatorImpl* allocator, size_t amount);
   // Finish reclamation pass.
   void FinishReclamation(uint64_t token, Waker waker);
   // Return some memory to the quota.
   void Return(size_t amount);
+  // Add allocator to list of allocators in small bucket. Returns allocator id.
+  void AddNewAllocator(GrpcMemoryAllocatorImpl* allocator);
+  // Remove allocator from list of allocators.
+  void RemoveAllocator(GrpcMemoryAllocatorImpl* allocator);
+  // Determine whether to move allocator to different bucket and if so, move.
+  void MaybeMoveAllocator(GrpcMemoryAllocatorImpl* allocator,
+                          size_t old_free_bytes, size_t new_free_bytes);
   // Instantaneous memory pressure approximation.
   PressureInfo GetPressureInfo();
   // Get a reclamation queue
@@ -319,8 +335,29 @@ class BasicMemoryQuota final
   friend class ReclamationSweep;
   class WaitForSweepPromise;
 
+  class AllocatorBucket {
+   public:
+    struct Shard {
+      absl::flat_hash_set<GrpcMemoryAllocatorImpl*> allocators
+          ABSL_GUARDED_BY(shard_mu);
+      absl::Mutex shard_mu;
+    };
+
+    Shard& SelectShard(void* key) {
+      const size_t hash = HashPointer(key, shards.size());
+      return shards[hash % shards.size()];
+    }
+
+    std::array<Shard, 16> shards;
+  };
+
   static constexpr intptr_t kInitialSize = std::numeric_limits<intptr_t>::max();
 
+  // Move allocator from big bucket to small bucket.
+  void MaybeMoveAllocatorBigToSmall(GrpcMemoryAllocatorImpl* allocator);
+  // Move allocator from small bucket to big bucket.
+  void MaybeMoveAllocatorSmallToBig(GrpcMemoryAllocatorImpl* allocator);
+
   // The amount of memory that's free in this quota.
   // We use intptr_t as a reasonable proxy for ssize_t that's portable.
   // We allow arbitrary overcommit and so this must allow negative values.
@@ -330,6 +367,10 @@ class BasicMemoryQuota final
 
   // Reclaimer queues.
   ReclaimerQueue reclaimers_[kNumReclamationPasses];
+  // List of all allocators sorted into 2 buckets, small (<100 KB free bytes)
+  // and large (>500 KB free bytes).
+  AllocatorBucket small_allocators_;
+  AllocatorBucket big_allocators_;
   // The reclaimer activity consumes reclaimers whenever we are in overcommit to
   // try and get back under memory limits.
   ActivityPtr reclaimer_activity_;
@@ -367,13 +408,25 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl {
     size_t prev_free = free_bytes_.fetch_add(n, std::memory_order_release);
     if ((!IsUnconstrainedMaxQuotaBufferSizeEnabled() &&
          prev_free + n > kMaxQuotaBufferSize) ||
-        (IsPeriodicResourceQuotaReclamationEnabled() &&
-         donate_back_.Tick([](Duration) {}))) {
+        donate_back_.Tick([](Duration) {})) {
       // Try to immediately return some free'ed memory back to the total quota.
       MaybeDonateBack();
     }
-    if (prev_free != 0) return;
-    MaybeRegisterReclaimer();
+    size_t new_free = free_bytes_.load(std::memory_order_relaxed);
+    memory_quota_->MaybeMoveAllocator(this, prev_free, new_free);
+  }
+
+  // Return all free bytes to quota.
+  void ReturnFree() {
+    size_t ret = free_bytes_.exchange(0, std::memory_order_acq_rel);
+    if (ret == 0) return;
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
+      gpr_log(GPR_INFO, "Allocator %p returning %zu bytes to quota", this, ret);
+    }
+    taken_bytes_.fetch_sub(ret, std::memory_order_relaxed);
+    memory_quota_->Return(ret);
+    memory_quota_->MaybeMoveAllocator(this, /*old_free_bytes=*/ret,
+                                      /*new_free_bytes=*/0);
   }
 
   // Post a reclamation function.
@@ -395,8 +448,17 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl {
   // Name of this allocator
   absl::string_view name() const { return name_; }
 
+  size_t GetFreeBytes() const {
+    return free_bytes_.load(std::memory_order_relaxed);
+  }
+
+  size_t IncrementShardIndex() {
+    return chosen_shard_idx_.fetch_add(1, std::memory_order_relaxed);
+  }
+
  private:
   static constexpr size_t kMaxQuotaBufferSize = 1024 * 1024;
+
   // Primitive reservation function.
   absl::optional<size_t> TryReserve(MemoryRequest request) GRPC_MUST_USE_RESULT;
   // This function may be invoked during a memory release operation.
@@ -405,9 +467,6 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl {
   // Replenish bytes from the quota, without blocking, possibly entering
   // overcommit.
   void Replenish();
-  // If we have not already, register a reclamation function against the quota
-  // to sweep any free memory back to that quota.
-  void MaybeRegisterReclaimer() ABSL_LOCKS_EXCLUDED(reclaimer_mu_);
   template <typename F>
   void InsertReclaimer(size_t pass, F fn)
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(reclaimer_mu_) {
@@ -424,7 +483,8 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl {
   std::atomic<size_t> free_bytes_{0};
   // Amount of memory taken from the quota by this allocator.
   std::atomic<size_t> taken_bytes_{sizeof(GrpcMemoryAllocatorImpl)};
-  std::atomic<bool> registered_reclaimer_{false};
+  // Index used to randomly choose shard to return bytes from.
+  std::atomic<size_t> chosen_shard_idx_{0};
   // We try to donate back some memory periodically to the central quota.
   PeriodicUpdate donate_back_{Duration::Seconds(10)};
   Mutex reclaimer_mu_;
@@ -434,6 +494,7 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl {
   OrphanablePtr<ReclaimerQueue::Handle>
       reclamation_handles_[kNumReclamationPasses] ABSL_GUARDED_BY(
           reclaimer_mu_);
+
   // Name of this allocator.
   std::string name_;
 };
@@ -461,6 +522,7 @@ class MemoryOwner final : public MemoryAllocator {
 
   // Instantaneous memory pressure in the underlying quota.
   BasicMemoryQuota::PressureInfo GetPressureInfo() const {
+    if (!is_valid()) return {};
     return impl()->GetPressureInfo();
   }
 
@@ -508,10 +570,10 @@ class MemoryQuota final
   // Resize the quota to new_size.
   void SetSize(size_t new_size) { memory_quota_->SetSize(new_size); }
 
-  // Return true if the instantaneous memory pressure is high.
+  // Return true if the controlled memory pressure is high.
   bool IsMemoryPressureHigh() const {
-    static constexpr double kMemoryPressureHighThreshold = 1.0;
-    return memory_quota_->GetPressureInfo().instantaneous_pressure >
+    static constexpr double kMemoryPressureHighThreshold = 0.99;
+    return memory_quota_->GetPressureInfo().pressure_control_value >
            kMemoryPressureHighThreshold;
   }
 

data/src/core/lib/resource_quota/resource_quota.h

@@ -22,7 +22,7 @@
 
 #include "absl/strings/string_view.h"
 
-#include <grpc/impl/codegen/grpc_types.h>
+#include <grpc/grpc.h>
 
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/cpp_impl_of.h"

data/src/core/lib/security/authorization/authorization_policy_provider.h

@@ -19,8 +19,8 @@
 
 #include "absl/strings/string_view.h"
 
+#include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
-#include <grpc/impl/codegen/grpc_types.h>
 
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/dual_ref_counted.h"

data/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc

@@ -16,7 +16,6 @@
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
-#include <grpc/impl/codegen/grpc_types.h>
 
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"

data/src/core/lib/security/authorization/rbac_policy.h

@@ -175,4 +175,4 @@ struct Rbac {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_SECURITY_AUTHORIZATION_RBAC_POLICY_H */
+#endif  // GRPC_CORE_LIB_SECURITY_AUTHORIZATION_RBAC_POLICY_H