grpc 1.45.0 → 1.53.0

data/src/core/lib/resource_quota/memory_quota.h

@@ -12,31 +12,48 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_RESOURCE_QUOTA_MEMORY_QUOTA_H
-#define GRPC_CORE_LIB_RESOURCE_QUOTA_MEMORY_QUOTA_H
+#ifndef GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_MEMORY_QUOTA_H
+#define GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_MEMORY_QUOTA_H
 
 #include <grpc/support/port_platform.h>
 
-#include <algorithm>
+#include <stdint.h>
+
+#include <array>
 #include <atomic>
 #include <cstddef>
 #include <limits>
 #include <memory>
-#include <queue>
-#include <vector>
+#include <string>
+#include <utility>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/container/flat_hash_set.h"
+#include "absl/strings/string_view.h"
+#include "absl/synchronization/mutex.h"
+#include "absl/types/optional.h"
 
 #include <grpc/event_engine/memory_allocator.h>
-#include <grpc/slice.h>
+#include <grpc/event_engine/memory_request.h>
+#include <grpc/support/log.h>
 
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/experiments/experiments.h"
+#include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/gprpp/time.h"
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/poll.h"
+#include "src/core/lib/resource_quota/periodic_update.h"
+#include "src/core/lib/resource_quota/trace.h"
 
 namespace grpc_core {
 
 class BasicMemoryQuota;
 class MemoryQuota;
+class GrpcMemoryAllocatorImpl;
 
 using grpc_event_engine::experimental::MemoryRequest;
 
@@ -53,26 +70,22 @@ using Vector = grpc_event_engine::experimental::Vector<T>;
 // reclaimers. We do this in multiple passes: if there is a less destructive
 // operation available, we do that, otherwise we do something more destructive.
 enum class ReclamationPass {
-  // Non-empty reclamation ought to take index 0, but to simplify API we don't
-  // expose that publicly (it's an internal detail), and hence index zero is
-  // here unnamed.
-
   // Benign reclamation is intended for reclamation steps that are not
   // observable outside of gRPC (besides maybe causing an increase in CPU
   // usage).
   // Examples of such reclamation would be resizing buffers to fit the current
   // load needs, rather than whatever was the peak usage requirement.
-  kBenign = 1,
+  kBenign = 0,
   // Idle reclamation is intended for reclamation steps that are observable
   // outside of gRPC, but do not cause application work to be lost.
   // Examples of such reclamation would be dropping channels that are not being
   // used.
-  kIdle = 2,
+  kIdle = 1,
   // Destructive reclamation is our last resort, and is these reclamations are
   // allowed to drop work - such as cancelling in flight requests.
-  kDestructive = 3,
+  kDestructive = 2,
 };
-static constexpr size_t kNumReclamationPasses = 4;
+static constexpr size_t kNumReclamationPasses = 3;
 
 // For each reclamation function run we construct a ReclamationSweep.
 // When this object is finally destroyed (it may be moved several times first),
@@ -212,9 +225,78 @@ class ReclaimerQueue {
   std::shared_ptr<State> state_;
 };
 
+namespace memory_quota_detail {
+// Controller: tries to adjust a control variable up or down to get memory
+// pressure to some target. We use the control variable to size buffers
+// throughout the stack.
+class PressureController {
+ public:
+  PressureController(uint8_t max_ticks_same, uint8_t max_reduction_per_tick)
+      : max_ticks_same_(max_ticks_same),
+        max_reduction_per_tick_(max_reduction_per_tick) {}
+  // Update the controller, returns the new control value.
+  double Update(double error);
+  // Textual representation of the controller.
+  std::string DebugString() const;
+
+ private:
+  // How many update periods have we reached the same decision in a row?
+  // Too many and we should start expanding the search space since we're not
+  // being agressive enough.
+  uint8_t ticks_same_ = 0;
+  // Maximum number of ticks with the same value until we start expanding the
+  // control space.
+  const uint8_t max_ticks_same_;
+  // Maximum amount to reduce the reporting value per iteration (in tenths of a
+  // percentile).
+  const uint8_t max_reduction_per_tick_;
+  // Was the last error indicating a too low pressure (or if false,
+  // a too high pressure).
+  bool last_was_low_ = true;
+  // Current minimum value to report.
+  double min_ = 0.0;
+  // Current maximum value to report.
+  // Set so that the first change over will choose 1.0 for max.
+  double max_ = 2.0;
+  // Last control value reported.
+  double last_control_ = 0.0;
+};
+
+// Utility to track memory pressure.
+// Tries to be conservative (returns a higher pressure than there may actually
+// be) but to be eventually accurate.
+class PressureTracker {
+ public:
+  double AddSampleAndGetControlValue(double sample);
+
+ private:
+  std::atomic<double> max_this_round_{0.0};
+  std::atomic<double> report_{0.0};
+  PeriodicUpdate update_{Duration::Seconds(1)};
+  PressureController controller_{100, 3};
+};
+}  // namespace memory_quota_detail
+
+// Minimum number of free bytes in order for allocator to move to big bucket.
+static constexpr size_t kBigAllocatorThreshold = 0.5 * 1024 * 1024;
+// Maximum number of free bytes in order for allocator to move to small
+// bucket.
+static constexpr size_t kSmallAllocatorThreshold = 0.1 * 1024 * 1024;
+
 class BasicMemoryQuota final
     : public std::enable_shared_from_this<BasicMemoryQuota> {
  public:
+  // Data about current memory pressure.
+  struct PressureInfo {
+    // The current instantaneously measured memory pressure.
+    double instantaneous_pressure = 0.0;
+    // A control value that can be used to scale buffer sizes up or down to
+    // adjust memory pressure to our target set point.
+    double pressure_control_value = 0.0;
+    // Maximum recommended individual allocation size.
+    size_t max_recommended_allocation_size = 0;
+  };
+
   explicit BasicMemoryQuota(std::string name) : name_(std::move(name)) {}
 
   // Start the reclamation activity.
@@ -229,14 +311,20 @@ class BasicMemoryQuota final
   void SetSize(size_t new_size);
   // Forcefully take some memory from the quota, potentially entering
   // overcommit.
-  void Take(size_t amount);
+  void Take(GrpcMemoryAllocatorImpl* allocator, size_t amount);
   // Finish reclamation pass.
   void FinishReclamation(uint64_t token, Waker waker);
   // Return some memory to the quota.
   void Return(size_t amount);
+  // Add allocator to list of allocators in small bucket. Returns allocator id.
+  void AddNewAllocator(GrpcMemoryAllocatorImpl* allocator);
+  // Remove allocator from list of allocators.
+  void RemoveAllocator(GrpcMemoryAllocatorImpl* allocator);
+  // Determine whether to move allocator to different bucket and if so, move.
+  void MaybeMoveAllocator(GrpcMemoryAllocatorImpl* allocator,
+                          size_t old_free_bytes, size_t new_free_bytes);
   // Instantaneous memory pressure approximation.
-  std::pair<double, size_t>
-  InstantaneousPressureAndMaxRecommendedAllocationSize() const;
+  PressureInfo GetPressureInfo();
   // Get a reclamation queue
   ReclaimerQueue* reclaimer_queue(size_t i) { return &reclaimers_[i]; }
 
@@ -247,8 +335,29 @@ class BasicMemoryQuota final
   friend class ReclamationSweep;
   class WaitForSweepPromise;
 
+  class AllocatorBucket {
+   public:
+    struct Shard {
+      absl::flat_hash_set<GrpcMemoryAllocatorImpl*> allocators
+          ABSL_GUARDED_BY(shard_mu);
+      absl::Mutex shard_mu;
+    };
+
+    Shard& SelectShard(void* key) {
+      const size_t hash = HashPointer(key, shards.size());
+      return shards[hash % shards.size()];
+    }
+
+    std::array<Shard, 16> shards;
+  };
+
   static constexpr intptr_t kInitialSize = std::numeric_limits<intptr_t>::max();
 
+  // Move allocator from big bucket to small bucket.
+  void MaybeMoveAllocatorBigToSmall(GrpcMemoryAllocatorImpl* allocator);
+  // Move allocator from small bucket to big bucket.
+  void MaybeMoveAllocatorSmallToBig(GrpcMemoryAllocatorImpl* allocator);
+
   // The amount of memory that's free in this quota.
   // We use intptr_t as a reasonable proxy for ssize_t that's portable.
   // We allow arbitrary overcommit and so this must allow negative values.
@@ -258,6 +367,10 @@ class BasicMemoryQuota final
 
   // Reclaimer queues.
   ReclaimerQueue reclaimers_[kNumReclamationPasses];
+  // List of all allocators sorted into 2 buckets, small (<100 KB free bytes)
+  // and large (>500 KB free bytes).
+  AllocatorBucket small_allocators_;
+  AllocatorBucket big_allocators_;
   // The reclaimer activity consumes reclaimers whenever we are in overcommit to
   // try and get back under memory limits.
   ActivityPtr reclaimer_activity_;
@@ -268,6 +381,8 @@ class BasicMemoryQuota final
   // We also increment this counter on completion of a sweep, as an indicator
   // that the wait has ended.
   std::atomic<uint64_t> reclamation_counter_{0};
+  // Memory pressure smoothing
+  memory_quota_detail::PressureTracker pressure_tracker_;
   // The name of this quota - used for debugging/tracing/etc..
   std::string name_;
 };
@@ -280,11 +395,6 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl {
       std::shared_ptr<BasicMemoryQuota> memory_quota, std::string name);
   ~GrpcMemoryAllocatorImpl() override;
 
-  // Rebind - Swaps the underlying quota for this allocator, taking care to
-  // make sure memory allocated is moved to allocations against the new quota.
-  void Rebind(std::shared_ptr<BasicMemoryQuota> memory_quota)
-      ABSL_LOCKS_EXCLUDED(memory_quota_mu_);
-
   // Reserve bytes from the quota.
   // If we enter overcommit, reclamation will begin concurrently.
   // Returns the number of bytes reserved.
@@ -295,14 +405,34 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl {
     // Add the released memory to our free bytes counter... if this increases
     // from  0 to non-zero, then we have more to do, otherwise, we're actually
     // done.
-    if (free_bytes_.fetch_add(n, std::memory_order_release) != 0) return;
-    MaybeRegisterReclaimer();
+    size_t prev_free = free_bytes_.fetch_add(n, std::memory_order_release);
+    if ((!IsUnconstrainedMaxQuotaBufferSizeEnabled() &&
+         prev_free + n > kMaxQuotaBufferSize) ||
+        donate_back_.Tick([](Duration) {})) {
+      // Try to immediately return some free'ed memory back to the total quota.
+      MaybeDonateBack();
+    }
+    size_t new_free = free_bytes_.load(std::memory_order_relaxed);
+    memory_quota_->MaybeMoveAllocator(this, prev_free, new_free);
+  }
+
+  // Return all free bytes to quota.
+  void ReturnFree() {
+    size_t ret = free_bytes_.exchange(0, std::memory_order_acq_rel);
+    if (ret == 0) return;
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
+      gpr_log(GPR_INFO, "Allocator %p returning %zu bytes to quota", this, ret);
+    }
+    taken_bytes_.fetch_sub(ret, std::memory_order_relaxed);
+    memory_quota_->Return(ret);
+    memory_quota_->MaybeMoveAllocator(this, /*old_free_bytes=*/ret,
+                                      /*new_free_bytes=*/0);
   }
 
   // Post a reclamation function.
   template <typename F>
   void PostReclaimer(ReclamationPass pass, F fn) {
-    MutexLock lock(&memory_quota_mu_);
+    MutexLock lock(&reclaimer_mu_);
     GPR_ASSERT(!shutdown_);
     InsertReclaimer(static_cast<size_t>(pass), std::move(fn));
   }
@@ -311,53 +441,60 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl {
   void Shutdown() override;
 
   // Read the instantaneous memory pressure
-  double InstantaneousPressure() const {
-    MutexLock lock(&memory_quota_mu_);
-    return memory_quota_->InstantaneousPressureAndMaxRecommendedAllocationSize()
-        .first;
+  BasicMemoryQuota::PressureInfo GetPressureInfo() const {
+    return memory_quota_->GetPressureInfo();
   }
 
   // Name of this allocator
   absl::string_view name() const { return name_; }
 
+  size_t GetFreeBytes() const {
+    return free_bytes_.load(std::memory_order_relaxed);
+  }
+
+  size_t IncrementShardIndex() {
+    return chosen_shard_idx_.fetch_add(1, std::memory_order_relaxed);
+  }
+
  private:
+  static constexpr size_t kMaxQuotaBufferSize = 1024 * 1024;
+
   // Primitive reservation function.
   absl::optional<size_t> TryReserve(MemoryRequest request) GRPC_MUST_USE_RESULT;
+  // This function may be invoked during a memory release operation.
+  // It will try to return half of our free pool to the quota.
+  void MaybeDonateBack();
   // Replenish bytes from the quota, without blocking, possibly entering
   // overcommit.
-  void Replenish() ABSL_LOCKS_EXCLUDED(memory_quota_mu_);
-  // If we have not already, register a reclamation function against the quota
-  // to sweep any free memory back to that quota.
-  void MaybeRegisterReclaimer() ABSL_LOCKS_EXCLUDED(memory_quota_mu_);
-  void MaybeRegisterReclaimerLocked()
-      ABSL_EXCLUSIVE_LOCKS_REQUIRED(memory_quota_mu_);
+  void Replenish();
   template <typename F>
   void InsertReclaimer(size_t pass, F fn)
-      ABSL_EXCLUSIVE_LOCKS_REQUIRED(memory_quota_mu_) {
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(reclaimer_mu_) {
     reclamation_handles_[pass] =
         memory_quota_->reclaimer_queue(pass)->Insert(std::move(fn));
   }
 
+  // Backing resource quota.
+  const std::shared_ptr<BasicMemoryQuota> memory_quota_;
   // Amount of memory this allocator has cached for its own use: to avoid quota
   // contention, each MemoryAllocator can keep some memory in addition to what
   // it is immediately using, and the quota can pull it back under memory
   // pressure.
   std::atomic<size_t> free_bytes_{0};
-  // Mutex guarding the backing resource quota.
-  mutable Mutex memory_quota_mu_;
-  // Backing resource quota.
-  std::shared_ptr<BasicMemoryQuota> memory_quota_
-      ABSL_GUARDED_BY(memory_quota_mu_);
   // Amount of memory taken from the quota by this allocator.
-  size_t taken_bytes_ ABSL_GUARDED_BY(memory_quota_mu_) =
-      sizeof(GrpcMemoryAllocatorImpl);
-  bool shutdown_ ABSL_GUARDED_BY(memory_quota_mu_) = false;
-  bool registered_reclaimer_ ABSL_GUARDED_BY(memory_quota_mu_) = false;
+  std::atomic<size_t> taken_bytes_{sizeof(GrpcMemoryAllocatorImpl)};
+  // Index used to randomly choose shard to return bytes from.
+  std::atomic<size_t> chosen_shard_idx_{0};
+  // We try to donate back some memory periodically to the central quota.
+  PeriodicUpdate donate_back_{Duration::Seconds(10)};
+  Mutex reclaimer_mu_;
+  bool shutdown_ ABSL_GUARDED_BY(reclaimer_mu_) = false;
   // Indices into the various reclaimer queues, used so that we can cancel
   // reclamation should we shutdown or get rebound.
   OrphanablePtr<ReclaimerQueue::Handle>
       reclamation_handles_[kNumReclamationPasses] ABSL_GUARDED_BY(
-          memory_quota_mu_);
+          reclaimer_mu_);
+
   // Name of this allocator.
   std::string name_;
 };
@@ -383,12 +520,10 @@ class MemoryOwner final : public MemoryAllocator {
     impl()->PostReclaimer(pass, std::move(fn));
   }
 
-  // Rebind to a different quota.
-  void Rebind(MemoryQuota* quota);
-
   // Instantaneous memory pressure in the underlying quota.
-  double InstantaneousPressure() const {
-    return impl()->InstantaneousPressure();
+  BasicMemoryQuota::PressureInfo GetPressureInfo() const {
+    if (!is_valid()) return {};
+    return impl()->GetPressureInfo();
   }
 
   template <typename T, typename... Args>
@@ -435,11 +570,11 @@ class MemoryQuota final
   // Resize the quota to new_size.
   void SetSize(size_t new_size) { memory_quota_->SetSize(new_size); }
 
-  // Return true if the instantaneous memory pressure is high.
+  // Return true if the controlled memory pressure is high.
   bool IsMemoryPressureHigh() const {
-    static constexpr double kMemoryPressureHighThreshold = 0.9;
-    return memory_quota_->InstantaneousPressureAndMaxRecommendedAllocationSize()
-               .first > kMemoryPressureHighThreshold;
+    static constexpr double kMemoryPressureHighThreshold = 0.99;
+    return memory_quota_->GetPressureInfo().pressure_control_value >
+           kMemoryPressureHighThreshold;
   }
 
  private:
@@ -454,4 +589,4 @@ inline MemoryQuotaRefPtr MakeMemoryQuota(std::string name) {
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_RESOURCE_QUOTA_MEMORY_QUOTA_H
+#endif  // GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_MEMORY_QUOTA_H

data/src/core/lib/resource_quota/periodic_update.cc

@@ -0,0 +1,78 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/resource_quota/periodic_update.h"
+
+#include <atomic>
+
+#include "src/core/lib/gpr/useful.h"
+
+namespace grpc_core {
+
+bool PeriodicUpdate::MaybeEndPeriod(absl::FunctionRef<void(Duration)> f) {
+  if (period_start_ == Timestamp::ProcessEpoch()) {
+    period_start_ = Timestamp::Now();
+    updates_remaining_.store(1, std::memory_order_release);
+    return false;
+  }
+  // updates_remaining_ just reached 0 and the thread calling this function was
+  // the decrementer that got us there.
+  // We can now safely mutate any non-atomic mutable variables (we've got a
+  // guarantee that no other thread will), and by the time this function returns
+  // we must store a postive number into updates_remaining_.
+  auto now = Timestamp::Now();
+  Duration time_so_far = now - period_start_;
+  if (time_so_far < period_) {
+    // At most double the number of updates remaining until the next period.
+    // At least try to estimate when we'll reach it.
+    int64_t better_guess;
+    if (time_so_far.millis() == 0) {
+      better_guess = expected_updates_per_period_ * 2;
+    } else {
+      // Determine a scaling factor that would have gotten us to the next
+      // period, but clamp between 1.01 (at least 1% increase in guesses)
+      // and 2.0 (at most doubling) - to avoid running completely out of
+      // control.
+      const double scale =
+          Clamp(period_.seconds() / time_so_far.seconds(), 1.01, 2.0);
+      better_guess = expected_updates_per_period_ * scale;
+      if (better_guess <= expected_updates_per_period_) {
+        better_guess = expected_updates_per_period_ + 1;
+      }
+    }
+    // Store the remainder left. Note that updates_remaining_ may have been
+    // decremented by another thread whilst we performed the above calculations:
+    // we simply discard those decrements.
+    updates_remaining_.store(better_guess - expected_updates_per_period_,
+                             std::memory_order_release);
+    // Not quite done, return, try for longer.
+    return false;
+  }
+  // Finished period, start a new one and return true.
+  // We try to predict how many update periods we'd need to cover the full time
+  // span, and we increase that by 1% to attempt to tend to not enter the above
+  // stanza.
+  expected_updates_per_period_ =
+      period_.seconds() * expected_updates_per_period_ / time_so_far.seconds();
+  if (expected_updates_per_period_ < 1) expected_updates_per_period_ = 1;
+  period_start_ = now;
+  f(time_so_far);
+  updates_remaining_.store(expected_updates_per_period_,
+                           std::memory_order_release);
+  return true;
+}
+
+}  // namespace grpc_core

data/src/core/lib/resource_quota/periodic_update.h

@@ -0,0 +1,71 @@
+// Copyright 2022 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_PERIODIC_UPDATE_H
+#define GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_PERIODIC_UPDATE_H
+
+#include <grpc/support/port_platform.h>
+
+#include <inttypes.h>
+
+#include <atomic>
+
+#include "absl/functional/function_ref.h"
+
+#include "src/core/lib/gprpp/time.h"
+
+namespace grpc_core {
+
+// Lightweight timer-like mechanism for periodic updates.
+// Fast path only decrements an atomic int64.
+// Slow path runs corrections and estimates how many ticks are required to hit
+// the target period.
+// This is super inaccurate of course, but for places where we can't run timers,
+// or places where continuous registration/unregistration would cause problems
+// it can be quite useful.
+class PeriodicUpdate {
+ public:
+  explicit PeriodicUpdate(Duration period) : period_(period) {}
+
+  // Tick the update, call f and return true if we think the period expired.
+  bool Tick(absl::FunctionRef<void(Duration)> f) {
+    // Atomically decrement the remaining ticks counter.
+    // If we hit 0 our estimate of period length has expired.
+    // See the comment next to the data members for a description of thread
+    // safety.
+    if (updates_remaining_.fetch_sub(1, std::memory_order_acquire) == 1) {
+      return MaybeEndPeriod(f);
+    }
+    return false;
+  }
+
+ private:
+  bool MaybeEndPeriod(absl::FunctionRef<void(Duration)> f);
+
+  // Thread safety:
+  // When updates_remaining_ reaches 0 the thread that decremented becomes
+  // responsible for updating any mutable variables and then setting
+  // updates_remaining_ to a value greater than zero.
+  // Whilst in this state other threads *may* decrement updates_remaining_, but
+  // this is fine because they'll observe an ignorable negative value.
+
+  std::atomic<int64_t> updates_remaining_{1};
+  const Duration period_;
+  Timestamp period_start_ = Timestamp::ProcessEpoch();
+  int64_t expected_updates_per_period_ = 1;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_PERIODIC_UPDATE_H

data/src/core/lib/resource_quota/resource_quota.h

@@ -12,20 +12,29 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_RESOURCE_QUOTA_RESOURCE_QUOTA_H
-#define GRPC_CORE_LIB_RESOURCE_QUOTA_RESOURCE_QUOTA_H
+#ifndef GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_RESOURCE_QUOTA_H
+#define GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_RESOURCE_QUOTA_H
 
 #include <grpc/support/port_platform.h>
 
-#include <grpc/impl/codegen/grpc_types.h>
+#include <string>
+#include <utility>
 
+#include "absl/strings/string_view.h"
+
+#include <grpc/grpc.h>
+
+#include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/gprpp/cpp_impl_of.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/resource_quota/memory_quota.h"
 #include "src/core/lib/resource_quota/thread_quota.h"
 
 namespace grpc_core {
 
 class ResourceQuota;
+
 using ResourceQuotaRefPtr = RefCountedPtr<ResourceQuota>;
 
 class ResourceQuota : public RefCounted<ResourceQuota>,
@@ -37,6 +46,8 @@ class ResourceQuota : public RefCounted<ResourceQuota>,
   ResourceQuota(const ResourceQuota&) = delete;
   ResourceQuota& operator=(const ResourceQuota&) = delete;
 
+  static absl::string_view ChannelArgName() { return GRPC_ARG_RESOURCE_QUOTA; }
+
   MemoryQuotaRefPtr memory_quota() { return memory_quota_; }
 
   const RefCountedPtr<ThreadQuota>& thread_quota() { return thread_quota_; }
@@ -44,6 +55,11 @@ class ResourceQuota : public RefCounted<ResourceQuota>,
   // The default global resource quota
   static ResourceQuotaRefPtr Default();
 
+  static int ChannelArgsCompare(const ResourceQuota* a,
+                                const ResourceQuota* b) {
+    return QsortCompare(a, b);
+  }
+
  private:
   MemoryQuotaRefPtr memory_quota_;
   RefCountedPtr<ThreadQuota> thread_quota_;
@@ -55,4 +71,4 @@ inline ResourceQuotaRefPtr MakeResourceQuota(std::string name) {
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_RESOURCE_QUOTA_RESOURCE_QUOTA_H
+#endif  // GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_RESOURCE_QUOTA_H

data/src/core/lib/resource_quota/thread_quota.cc

@@ -16,6 +16,8 @@
 
 #include "src/core/lib/resource_quota/thread_quota.h"
 
+#include <grpc/support/log.h>
+
 namespace grpc_core {
 
 ThreadQuota::ThreadQuota() = default;

data/src/core/lib/resource_quota/thread_quota.h

@@ -12,14 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_RESOURCE_QUOTA_THREAD_QUOTA_H
-#define GRPC_CORE_LIB_RESOURCE_QUOTA_THREAD_QUOTA_H
+#ifndef GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_THREAD_QUOTA_H
+#define GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_THREAD_QUOTA_H
 
 #include <grpc/support/port_platform.h>
 
 #include <cstddef>
+#include <limits>
+
+#include "absl/base/thread_annotations.h"
 
 #include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
 
 namespace grpc_core {
@@ -54,4 +58,4 @@ using ThreadQuotaPtr = RefCountedPtr<ThreadQuota>;
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_RESOURCE_QUOTA_THREAD_QUOTA_H
+#endif  // GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_THREAD_QUOTA_H

data/src/core/lib/resource_quota/trace.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_RESOURCE_QUOTA_TRACE_H
-#define GRPC_CORE_LIB_RESOURCE_QUOTA_TRACE_H
+#ifndef GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_TRACE_H
+#define GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_TRACE_H
 
 #include <grpc/support/port_platform.h>
 
@@ -21,4 +21,4 @@
 
 extern grpc_core::TraceFlag grpc_resource_quota_trace;
 
-#endif  // GRPC_CORE_LIB_RESOURCE_QUOTA_TRACE_H
+#endif  // GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_TRACE_H